From d429fb231b2ff15aa03147143a9f8a1b6c81857d Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 05:16:10 -0700 Subject: [PATCH 1/7] cpu: give the R10000 family its address widths The R10000 family has 44-bit virtual and 40-bit physical addresses, a FrameMask register and a wider EntryLo; iris kept the R4400's 40 and 36. The R4400 and R5000 are unchanged throughout. Virtual width. EntryHi, the TLB match and the xuseg bounds check all stopped at VA bit 39, but VPN2 is 43:13 and the user segment is 16 TiB (R10000 UM pp. 219, 288). A 64-bit kernel can put an n64 user stack above 40 bits (0x0fff_ffff_aee4, for one), and EntryHi dropped VA[43:40] so such an address took an Address Error. The width now comes from the part's VA_BITS in each place: the EntryHi write mask, the TLB's 64-bit compare (Tlb::set_va_bits, installed by the executor beside set_cache_attrs), the xuseg bounds check, the micro-TLB's cacheable segment test and the EntryHi a TLB exception loads. The mask was spelled three times and is now one const fn, mips_tlb::vpn2_mask. All of it is a const of the CPU model or set once at construction; the TLB lookup is unchanged. This reverses a comment in the executor (b7d2a18b) that said widening the EntryHi mask breaks the IP28 and Octane kernels. FrameMask (CP0 register 21) is new with the R10000. Bits 15:0 clear EntryLo bits 33:18 as a TLB write passes them on, and the rest of the register reads as zero (R10000 UM v2.0 p. 260). iris had no register 21, so a kernel that sets it to keep software flags out of the physical address got no effect. The core holds the register; the executor sets which bits are writable, all 16 on the R10000 and none on the R4400 and R5000. TLBWI and TLBWR pass the masked EntryLo values and TLBR returns the masked entry. The register is saved with the rest of CP0 and loads as zero from an older save. CpuModel::R10K_FAMILY says a part is in the R10000 family. Nothing changes while FrameMask is zero, its reset value. EntryLo holds a 28-bit PFN in bits 33:6, a 40-bit physical address, and the two-bit uncached attribute in bits 63:62; 61:34 are reserved (R10000 UM v2.0 pp. 239-240). iris kept 30 bits, so the attribute and PFN bits 33:30 read back as zero and never reached the TLB. The core has an EntryLo mask the executor sets per part: 0x3FFF_FFFF on the R4400 and R5000, 0xC000_0003_FFFF_FFFF on the R10000. The attribute is stored and read back only. Bits 33:32 are physical 39:38, where the Octane's guest keeps software flags; FrameMask clears them on a TLB write, so the page lands at its real address. A TLB write also clears the PFN bits below the page size, so TLBR does not return what was written to them (R10000 UM v2.0 pp. 299-300). The manual documents this for the R10000 only; the later parts follow it as a model choice. Translation already ignored those bits, so only TLBR changes. The Octane PROM's tlb diagnostic writes 0xa5a5a5a4 and 0x5a5a5a58 with 16 KB pages and expects 0xa5a5a524 and 0x5a5a5a18 back; it reported 320 compare errors. On the Octane, the IP28 and an O2 set to the R10000, EntryHi keeps 44 bits and mfc0 of EntryLo returns bits 63:62 and PFN bits 33:30 if the guest wrote them. Tests: a page at 0x123_4000_0000 is kept by the R10000 and masked, with an Address Error, by the R4400; and a TLB write on each, checking FrameMask, the wider EntryLo and the PFN bits below a 16 KB page through TLBR. --- src/cpu/mips_cache_v2.rs | 6 ++++ src/cpu/mips_core.rs | 47 +++++++++++++++++------- src/cpu/mips_exec.rs | 74 +++++++++++++++++++++++++------------- src/cpu/mips_exec_test.rs | 75 +++++++++++++++++++++++++++++++++++++++ src/cpu/mips_tlb.rs | 17 +++++++-- src/ip30_machine.rs | 5 +-- 6 files changed, 183 insertions(+), 41 deletions(-) diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 689fda0b8..ec4ecaf8b 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -565,6 +565,12 @@ pub trait CpuModel: MipsCache { /// and `Index_Load_Data` returns into it, neither of which is true of the /// R4000 hit operations that share those encodings. const R10K_CACHE_OPS: bool = false; + /// The part is in the R10000 family, which shares one programming model: + /// FrameMask, and 44 virtual and 40 physical address bits. + /// Established for the R10000. The R12000 is inferred to match: its manual's + /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and + /// R16000 are a model choice that gives them the same TLB. + const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; } pub trait MipsCache: Send + Sync { diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 068f7dab0..1643bf6d4 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -867,6 +867,8 @@ pub struct MipsCore { pub cp0_watchlo: u32, // 18: Watchpoint Low pub cp0_watchhi: u32, // 19: Watchpoint High pub cp0_xcontext: u64, // 20: Extended Context (64-bit) + pub cp0_framemask: u32, // 21: FrameMask (R10000 family), bits 15:0 clear EntryLo bits 33:18 on a TLB write + pub framemask_mask: u32, // Writable FrameMask bits: 15:0 on the R10000 family, none elsewhere pub cp0_ecc: u32, // 26: ECC Register pub cp0_cacheerr: u32, // 27: Cache Error /// 28: Cache Tag Low. 64 bits, not 32. @@ -881,6 +883,8 @@ pub struct MipsCore { pub cp0_errorepc: u64, // 30: Error Exception PC pub tlb_entries: u32, // Total TLB entries + pub entryhi_mask: u64, // Writable EntryHi bits: region, ASID and the model's VPN2 width + pub entrylo_mask: u64, // Writable EntryLo bits: 29:0, or on the R10000 family the PFN (33:6) and uncached attribute (63:62) /// CP0 PRId / CP1 FIR reset values, supplied by the CPU model at construction. pub reset_prid: u32, pub reset_fir: u32, @@ -1407,12 +1411,16 @@ impl MipsCore { cp0_watchlo: 0, cp0_watchhi: 0, cp0_xcontext: 0, + cp0_framemask: 0, + framemask_mask: 0, cp0_ecc: 0, cp0_cacheerr: 0, cp0_taglo: 0, cp0_taghi: 0, cp0_errorepc: 0, tlb_entries: 48, + entryhi_mask: 0xC000_00FF_FFFF_E0FF, + entrylo_mask: 0x3FFF_FFFF, reset_prid: 0x0000_0440, reset_fir: 0x0000_0500, cp0_random_cycle: 0, @@ -1498,6 +1506,7 @@ impl MipsCore { self.cp0_watchlo = 0; self.cp0_watchhi = 0; self.cp0_xcontext = 0; + self.cp0_framemask = 0; self.cp0_ecc = 0; self.cp0_cacheerr = 0; self.cp0_taglo = 0; @@ -1612,8 +1621,8 @@ impl MipsCore { self.update_random(); self.cp0_random as u64 } - 2 => self.cp0_entrylo0 & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits - 3 => self.cp0_entrylo1 & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits + 2 => self.cp0_entrylo0 & self.entrylo_mask, + 3 => self.cp0_entrylo1 & self.entrylo_mask, 4 => self.cp0_context, 5 => self.cp0_pagemask & 0x01FFE000, // PageMask: only bits 24:13 are valid 6 => self.cp0_wired as u64 & 0x3F, // Wired: only bits 5:0 are valid, R4000 manual Table 4-10 @@ -1636,6 +1645,7 @@ impl MipsCore { 18 => self.cp0_watchlo as u64, 19 => self.cp0_watchhi as u64, 20 => self.cp0_xcontext, + 21 => self.cp0_framemask as u64, 25 => { // PerfCnt: R4400 has no performance counters. Log any probe so // a guest expecting them (and routing their overflow onto the @@ -1670,8 +1680,8 @@ impl MipsCore { match reg { 0 => self.cp0_index as u64, 1 => self.cp0_random as u64, - 2 => self.cp0_entrylo0 & 0x3FFFFFFF, - 3 => self.cp0_entrylo1 & 0x3FFFFFFF, + 2 => self.cp0_entrylo0 & self.entrylo_mask, + 3 => self.cp0_entrylo1 & self.entrylo_mask, 4 => self.cp0_context, 5 => self.cp0_pagemask & 0x01FFE000, 6 => self.cp0_wired as u64 & 0x3F, @@ -1688,6 +1698,7 @@ impl MipsCore { 18 => self.cp0_watchlo as u64, 19 => self.cp0_watchhi as u64, 20 => self.cp0_xcontext, + 21 => self.cp0_framemask as u64, 26 => self.cp0_ecc as u64, 27 => self.cp0_cacheerr as u64, 28 => self.cp0_taglo, @@ -1697,6 +1708,14 @@ impl MipsCore { } } + /// An EntryLo value as a TLB write passes it on: FrameMask bits 15:0 clear + /// EntryLo bits 33:18, so the flags IRIX keeps above the PFN's low bits + /// never reach the physical address (R10000 UM v2.0 p. 260). + #[inline] + pub fn framemask_entrylo(&self, lo: u64) -> u64 { + lo & !((self.cp0_framemask as u64) << 18) + } + /// Materialize the virtual CP0 Count: advance `cp0_count` by the whole /// hardware ticks elapsed since the anchor at `count_hz`, move the anchor /// forward by exactly the duration those ticks consumed (the sub-tick @@ -2011,8 +2030,8 @@ impl MipsCore { 0 => self.cp0_index = (value as u32) & (crate::cpu::mips_exec::CP0_INDEX_P | crate::cpu::mips_exec::CP0_INDEX_SLOT_MASK), 1 => { /* Random is read-only */ } - 2 => self.cp0_entrylo0 = value & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits - 3 => self.cp0_entrylo1 = value & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits + 2 => self.cp0_entrylo0 = value & self.entrylo_mask, + 3 => self.cp0_entrylo1 = value & self.entrylo_mask, 4 => { //eprintln!("MTC0 Context = {:#018x} (PTEBase={:#018x})", value, value & 0xFFFFFFFF_FF800000); self.cp0_context = value; @@ -2035,16 +2054,16 @@ impl MipsCore { } 10 => { // always use 64bit mask because the entries need to be valid in 64 bit mode even when they were set from 32 bit mode // `log mips mask cp0` shows what the guest wrote against - // what survives the mask. The mask is R4400's 40-bit virtual - // address; the R10000 implements 44. - if cp0_log() && (value & !0xC000_00FF_FFFF_E0FF) != 0 { + // what survives the mask: the model's VPN2 width, 40 bits or the + // R10000 family's 44. + if cp0_log() && (value & !self.entryhi_mask) != 0 { crate::dlog!(crate::devlog::LogModule::Mips, "ip28ehi: wrote {value:#018x} -> kept {:#018x} (lost {:#018x}) pc={:#018x}", - value & 0xC000_00FF_FFFF_E0FF, - value & !0xC000_00FF_FFFF_E0FF, + value & self.entryhi_mask, + value & !self.entryhi_mask, self.pc); } - self.cp0_entryhi = value & 0xC000_00FF_FFFF_E0FF; + self.cp0_entryhi = value & self.entryhi_mask; }, 11 => { // Materialize the count *first* so both new_delta below and @@ -2253,6 +2272,10 @@ impl MipsCore { } self.cp0_xcontext = value; } + // FrameMask: bits 15:0 are implemented, the rest read as zero + // (R10000 UM v2.0 p. 260). Other parts have no register 21 and ignore + // the write. + 21 => self.cp0_framemask = value as u32 & self.framemask_mask, 25 => { #[cfg(feature = "developer_ip7")] eprintln!("[ip7] MTC0 PerfCnt (reg 25) write {:#018x} (ignored)", value); diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index cbc86ebcc..60204a0cb 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -577,17 +577,26 @@ pub const TR_GLOBAL: u32 = 1 << 4; /// the privilege epoch that filled it. The only thing rejected here is a VA /// shape that is not a valid address in the first place. #[inline] -fn nutlb_cacheable_segment(va: u64, is_64bit: bool) -> bool { +fn nutlb_cacheable_segment(va: u64, is_64bit: bool) -> bool { if !is_64bit { // Every 32-bit segment (kuseg/kseg0/kseg1/ksseg/kseg3) is a real // address whose finished translation is worth caching. return true; } - // xuseg: bits 63:40 must be zero to be a valid user address. Every other - // 64-bit segment (xsseg, xkphys, xkseg and the 0xFFFFFFFF_* compat - // windows) is fine — translate_64bit_impl raises before returning ok for - // the illegal xkphys sub-ranges, so nothing invalid reaches a fill. - (va >> 62) != 0 || (va >> 40) == 0 + // xuseg: bits 63:`C::VA_BITS` must be zero to be a valid user address. + // Every other 64-bit segment (xsseg, xkphys, xkseg and the 0xFFFFFFFF_* + // compat windows) is fine — translate_64bit_impl raises before returning ok + // for the illegal xkphys sub-ranges, so nothing invalid reaches a fill. + (va >> 62) != 0 || (va >> C::VA_BITS) == 0 +} + +/// PageMask bit n covers physical address bit n-1, which is EntryLo bit n-7. +const PAGEMASK_TO_ENTRYLO: u32 = 7; + +/// The EntryHi bits this model keeps: region, VPN2 up to `C::VA_BITS` and ASID. +#[inline(always)] +const fn model_entryhi_mask() -> u64 { + 0xC000_0000_0000_00FF | crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS) } /// Per-access-class tally of how far a guest load/store would get through the @@ -2215,7 +2224,15 @@ impl MipsExecutor { let mut core = MipsCore::new(); // Mapped pages decode their C field with this model's table, as KSEG0 // and xkphys do. + tlb.set_va_bits(C::VA_BITS); tlb.set_cache_attrs(Self::cache_attr_table()); + core.entryhi_mask = model_entryhi_mask::(); + if C::R10K_FAMILY { + core.framemask_mask = 0xFFFF; + // PFN in bits 33:6 and the uncached attribute in 63:62 (R10000 UM v2.0 + // pp. 239-240); 61:34 are reserved. + core.entrylo_mask = 0xC000_0003_FFFF_FFFF; + } // Cache geometry comes from the model type C, so Config describes this CPU. // `mut` is only used by the Triton L2-enable sync below. @@ -2890,7 +2907,7 @@ impl MipsExecutor { // user address. Everything else is cacheable regardless of which // mode filled it, because privilege is enforced by flushing on // transition rather than by the tag. - if !nutlb_cacheable_segment(va, self.core.is_64bit_mode()) { + if !nutlb_cacheable_segment::(va, self.core.is_64bit_mode()) { return; } @@ -4548,11 +4565,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let top_bits = virt_addr >> 62; match top_bits { - // xuseg: 0x0000_0000_0000_0000 - 0x0000_00FF_FFFF_FFFF (user mapped) - // Accessible from all privilege levels + // xuseg is user-mapped below 2^C::VA_BITS (0x0000_00FF_FFFF_FFFF + // for the 40-bit models) and accessible from every privilege level. 0 => { - if (virt_addr >> 40) != 0 { - // Bits 63:40 must be zero for valid user address + if (virt_addr >> C::VA_BITS) != 0 { + // Bits above the model's implemented user VA must be zero if !DEBUG { self.core.cp0_badvaddr = virt_addr; } return TranslateResult::exc(exec_exception(if access_type == AccessType::Write { EXC_ADES } else { EXC_ADEL })); } @@ -4703,8 +4720,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[inline] fn update_tlb_exception_registers(&mut self, virt_addr: u64) { const EH_VPN2_32: u64 = 0x0000_0000_FFFF_E000; - const EH_VPN2_64: u64 = 0x0000_00FF_FFFF_E000; const EH_REGION: u64 = 0xC000_0000_0000_0000; + // VPN2 is bits 43:13 on the R10000 family (R10000 UM pp. 219, 288) and + // 39:13 on the R4x00 and R5000. The R12000 is inferred to match (its + // manual lists no TLB change); the R14000 and R16000 are a model choice + // that follows the same width. + let eh_vpn2_64 = crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS); self.core.cp0_badvaddr = virt_addr; @@ -4727,9 +4748,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // // This is MAME r4000's `EH_WM` (0xC000_00FF_FFFF_E0FF) less the ASID // bits, which are carried separately below. - const EH_VPN_MASK: u64 = EH_REGION | EH_VPN2_64; // 0xC000_00FF_FFFF_E000 + let eh_vpn_mask = EH_REGION | eh_vpn2_64; let asid = self.core.cp0_entryhi & 0xFF; - self.core.cp0_entryhi = (virt_addr & EH_VPN_MASK) | asid; + self.core.cp0_entryhi = (virt_addr & eh_vpn_mask) | asid; // Context: PTEBase[63:23] preserved, BadVPN2 = virt_addr[31:13] in bits [22:4]. // Always 32-bit VPN — Context is used by the 32-bit UTLB handler. @@ -4750,10 +4771,6 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // the address it computes lands outside the page table it wired, the // nested miss is unrecoverable, and IRIX panics with "Entered tlbmiss // with invalid vaddr". - // - // EntryHi keeps the 40-bit VPN2 field on every model: a 64-bit kernel - // zero-fills those upper bits itself when it builds an EntryHi value, - // and widening the mask here breaks it. self.core.cp0_xcontext = xcontext_on_miss(C::VA_BITS, self.core.cp0_xcontext, virt_addr); } @@ -7159,13 +7176,20 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let g1 = (self.core.cp0_entrylo1 & 1) != 0; let g_combined = if g0 && g1 { 1u64 << 12 } else { 0 }; - // EH_WM = 0xC000_00FF_FFFF_E0FF — same mask MAME applies on TLBWI (clears reserved bits). - // Then clear bit 12 (G bit from EntryHi) and set combined G from Lo0&Lo1. - const EH_WM: u64 = 0xC000_00FF_FFFF_E0FF; + // EntryHi's writable mask (0xC000_00FF_FFFF_E0FF on a 40-bit model, MAME's + // `EH_WM`) clears reserved bits, as on TLBWI. Then clear bit 12 (G bit + // from EntryHi) and set combined G from Lo0&Lo1. + // + // The R10000 family stores zeros in the PFN bits the page size leaves + // out of the address, so TLBR does not return them (R10000 UM v2.0 + // pp. 299-300). The manual documents this for the R10000 only; the + // later parts follow it as a model choice. + let unused_pfn = if C::R10K_FAMILY { (self.core.cp0_pagemask & 0x01FF_E000) >> PAGEMASK_TO_ENTRYLO } else { 0 }; + let lo = |v: u64| self.core.framemask_entrylo(v) & !unused_pfn; TlbEntry { page_mask: self.core.cp0_pagemask, - entry_hi: (self.core.cp0_entryhi & EH_WM & !0x1000) | g_combined, - entry_lo: [self.core.cp0_entrylo0, self.core.cp0_entrylo1], + entry_hi: (self.core.cp0_entryhi & self.core.entryhi_mask & !0x1000) | g_combined, + entry_lo: [lo(self.core.cp0_entrylo0), lo(self.core.cp0_entrylo1)], selector_bit_shift: 0, // all derived fields overwritten by MipsTlb::write() vcmp32: 0, vpn_hi32: 0, vcmp64: 0, vpn_hi64: 0, offset_mask: 0, pfn_base: [0; 2], @@ -11069,7 +11093,7 @@ impl Saveable for MipsCpu cp0u64!(cp0_taglo); cp0u32!(cp0_taghi); cp0u64!(cp0_badvaddr); cp0u64!(cp0_epc); cp0u64!(cp0_errorepc); cp0u64!(cp0_entrylo0); cp0u64!(cp0_entrylo1); cp0u64!(cp0_context); - cp0u64!(cp0_pagemask); cp0u64!(cp0_entryhi); cp0u64!(cp0_xcontext); + cp0u64!(cp0_pagemask); cp0u64!(cp0_entryhi); cp0u64!(cp0_xcontext); cp0u32!(cp0_framemask); tbl.insert("cp0".into(), toml::Value::Table(cp0)); // FPU @@ -11134,7 +11158,7 @@ impl Saveable for MipsCpu ld32!(cp0_ecc); ld32!(cp0_cacheerr); ld64!(cp0_taglo); ld32!(cp0_taghi); ld64!(cp0_entrylo0); ld64!(cp0_entrylo1); ld64!(cp0_context); ld64!(cp0_pagemask); ld64!(cp0_badvaddr); ld64!(cp0_entryhi); - ld64!(cp0_xcontext); ld64!(cp0_epc); ld64!(cp0_errorepc); + ld64!(cp0_xcontext); ld32!(cp0_framemask); ld64!(cp0_epc); ld64!(cp0_errorepc); // Restart the virtual count from the restored value and re-arm // the compare timer — Instants from the previous run are // meaningless here — with a live ticket for the restored deadline. diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index c9cce670c..b7c4ff687 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6492,4 +6492,79 @@ mod tests { } assert_eq!(outcome[0], outcome[1], "an explanation must not change what the guest sees"); } + + type TlbExec = Box>; + + fn on_big_stack(f: impl FnOnce() + Send + 'static) { + std::thread::Builder::new().stack_size(16 << 20).spawn(f).unwrap().join().unwrap(); // MipsTlb is large + } + + /// A 64-bit kernel executor with a TLB, over `bus`. + fn tlb_exec>>(bus: Arc) -> TlbExec { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::new(bus, crate::cpu::mips_tlb::MipsTlb::new(C::TLB_ENTRIES), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + exec.set_cp0_status((exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX); + exec + } + + /// Write CP0 registers `regs` (register, value), then run the TLB op `funct`. + fn tlb_op(exec: &mut TlbExec, regs: &[(u32, u64)], funct: u32) { + for &(rd, value) in regs { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | funct), EXEC_COMPLETE); + } + + /// EntryHi and the TLB keep VPN2 bits 43:40, which the R4400's 40-bit + /// address space lacks. Fix: R10000 address widths. + #[test] + fn cp0_entryhi_and_tlb_match_full_xuseg_va() { + on_big_stack(|| { + const VA: u64 = 0x0000_0123_4000_0000; + fn check>>(full_va: bool) { + let mem = Arc::new(MockMemory::new()); + mem.set_word(0x5_0000, 0xdead_beef); + let mut exec = tlb_exec::(mem); + let lo0 = ((0x5_0000u64 >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + tlb_op(&mut exec, &[(10, VA), (0, 0), (5, 0), (2, lo0), (3, 0)], FUNCT_TLBWI); + let result = exec.read_data::<4>(VA); + if full_va { + assert_eq!((exec.core.cp0_entryhi, result), (VA, Ok(0xdead_beef))); + } else { + assert_eq!((result.unwrap_err() >> 2) & 0x1f, EXC_ADEL); + } + } + check::(false); + check::(true); + }); + } + + /// An R10000 TLB write clears the EntryLo bits FrameMask names (R10000 UM + /// v2.0 p. 260) and the PFN bits a 16 KB page leaves out (pp. 299-300), and + /// keeps the wider PFN and attribute; the R4400 has no FrameMask. + /// Fix: FrameMask, the wider EntryLo, TLBR's page-size masking. + #[test] + fn tlb_write_applies_framemask_and_page_size_on_the_r10000() { + use crate::cpu::mips_cache_shadow::{R4400ShadowCache, R10000ShadowCache}; + fn entry>>(framemask: u64) -> (u64, u64, u64) { + const VA: u64 = 0x4000_0000; + let lo0 = (0xC0_0300_4000u64 >> 12 << 6) | 0xC0 | (3 << 3) | (1 << 2) | (1 << 1) | (3 << 62); + let mut exec = tlb_exec::(Arc::new(MockMemory::new())); + tlb_op(&mut exec, &[(10, VA), (0, 0), (5, 0x6000), (21, framemask), (2, lo0), (3, 0)], FUNCT_TLBWI); + let phys = exec.debug_translate(VA).phys; + tlb_op(&mut exec, &[(10, VA), (2, 0)], FUNCT_TLBP); + tlb_op(&mut exec, &[], FUNCT_TLBR); + (exec.core.read_cp0(21), phys, exec.core.read_cp0(2)) + } + on_big_stack(|| { + let lo = (0xC0_0300_4000u64 >> 12 << 6) | (3 << 3) | (1 << 2) | (1 << 1); + // FrameMask 0xC000 is EntryLo 33:32; higher bits of the write are ignored. + assert_eq!(entry::(0xFFFF_C000), (0xC000, 0x0300_4000, (3 << 62) | (lo & !(3 << 32)))); + assert_eq!(entry::(0), (0, 0xC0_0300_4000, (3 << 62) | lo)); + assert_eq!(entry::(0xFFFF_C000), (0, 0x0300_4000, (lo & 0x3FFF_FFFF) | 0xC0)); + }); + } } diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 05680e0c6..275f513e0 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -295,6 +295,11 @@ pub enum TlbResult { }, } +/// VPN2 bits of a virtual address `bits` wide: bit 13 up to `bits - 1`. +pub const fn vpn2_mask(bits: u32) -> u64 { + ((1u64 << bits) - 1) & !0x1fff +} + /// TLB interface for MIPS address translation /// /// The TLB translates virtual page numbers to physical page numbers. @@ -351,6 +356,8 @@ pub trait Tlb { fn power_on(&mut self) {} /// Install the CPU model's meaning for each EntryLo C value. fn set_cache_attrs(&mut self, _table: [CacheAttr; 8]) {} + /// Set the implemented VPN width before installing entries. + fn set_va_bits(&mut self, _bits: u32) {} fn save_state(&self) -> toml::Value { toml::Value::Table(Default::default()) } fn load_state(&mut self, _v: &toml::Value) -> Result<(), String> { Ok(()) } @@ -534,6 +541,7 @@ pub struct MipsTlb { /// Live JTLB entries for the CPU model in use (<= TLB_MAX_ENTRIES). num_entries: usize, + vpn_mask64: u64, /// What each EntryLo C value means on the CPU model in use; the /// executor installs its table with `set_cache_attrs`. c_attr: [CacheAttr; 8], @@ -564,6 +572,7 @@ impl MipsTlb { mru_head: [0u8; MRU_LISTS], mru_next: [[MRU_NONE; TLB_MAX_ENTRIES]; MRU_LISTS], num_entries, + vpn_mask64: 0xC000_00FF_FFFF_E000, c_attr: DEFAULT_C_ATTR, vmap: [VMAP_MISS; VMAP_SIZE], #[cfg(feature = "tlbcheck")] @@ -1065,7 +1074,7 @@ impl Tlb for MipsTlb { entry.selector_bit_shift = (mask.trailing_ones() - 1) as u8; entry.vcmp32 = !mask & 0x0000_0000_FFFF_E000; entry.vpn_hi32 = entry.entry_hi & entry.vcmp32; - entry.vcmp64 = !mask & 0xC000_00FF_FFFF_E000; + entry.vcmp64 = !mask & self.vpn_mask64; entry.vpn_hi64 = entry.entry_hi & entry.vcmp64; entry.offset_mask = (1u64 << entry.selector_bit_shift) - 1; let pfn_mask = 0xFF_FFFF_FFFF_000 & !entry.offset_mask; @@ -1194,6 +1203,10 @@ impl Tlb for MipsTlb { output } + fn set_va_bits(&mut self, bits: u32) { + self.vpn_mask64 = 0xC000_0000_0000_0000 | vpn2_mask(bits); + } + fn set_cache_attrs(&mut self, table: [CacheAttr; 8]) { self.c_attr = table; for i in 0..self.num_entries { @@ -1237,7 +1250,7 @@ impl Tlb for MipsTlb { let entry_hi = words[1]; let mask = page_mask | 0x1FFF; let vcmp32 = !mask & 0x0000_0000_FFFF_E000; - let vcmp64 = !mask & 0xC000_00FF_FFFF_E000; + let vcmp64 = !mask & self.vpn_mask64; let shift = (mask.trailing_ones() - 1) as u8; let offset_mask = (1u64 << shift) - 1; let lo = [words[2], words[3]]; diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 4258b0d90..3aacdb09c 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -599,8 +599,9 @@ impl OctaneMachine { // IRIX zeroes Count and arms Compare a few instructions later; see // `MipsCore::count_write_samples`. exec.core.count_write_samples = true; - // The R10000 drives 40 physical address bits; HEART decodes them - // all, and the attribute bits above fold away. + // The R10000 drives 40 physical address bits; HEART decodes them all. + // EntryLo bits 33:32 are physical 39:38; a guest that keeps software + // flags there clears them with FrameMask, so no narrowing is needed. exec.phys_mask = (1 << 40) - 1; exec.core.cpu_id = id; exec.core.smp = cpu_count > 1; From a1cc9c74e330a1446e2fe19f5d3a2af031f5ea1e Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 1 Nov 2026 23:00:04 -0700 Subject: [PATCH 2/7] cpu: follow the R10000 rules for TLB refills and overlapping writes Refill vector. The R10000 picks the TLB refill vector from the address space the missing address lies in and that space's Status bit: user addresses follow UX, xsseg SX, and kseg3, sseg/ksseg and xkseg KX. The current mode only decides which space an address is in (R10000 UM v2.0 pp. 333-334, 342). The manual contrasts this with the R4400, which picks by the current mode's bit; iris did the R4400 thing on every model. So a 64-bit kernel (KX=1, UX=0) missing on a kuseg address went to the XTLB vector at 0x080 where the part uses 0x000. The Octane PROM's utlb diagnostic loads VA 0 in exactly that state and waits at the 32-bit vector; it got "Read TLB miss exception" from the XTLB vector and failed. On the R10000 family a refill miss now reads the region of the address, sign-extended from 32 bits in 32-bit mode, and tests UX, SX or KX. Only the miss path changes; the TLB compare width still follows the mode. Overlapping writes. The R10000 and R12000 do not shut down on a multiple match. A TLB write that would let one lookup match two entries first invalidates every entry it overlaps, the wired ones included, then writes the new entry and sets Status.TS (NEC U10278EJ4V0UM, Status register TS bit p.223 and "Avoiding TLB Conflict" p.300). iris stored the entry and left the duplicates matchable, so which one answered depended on how the lookup map happened to be rebuilt. On the two-processor Tezro the firmware flushes the TLB with 64 invalid entries on the kernel's page number, and the kernel then writes the valid global 16 MiB entry over them; a later write elsewhere left an invalid duplicate answering, and the fetch took TLB Invalid. The R14000 gets the same rule as a model choice; no manual for it was read. The R4400 and R5000 keep their behaviour. An overlap is two entries a lookup could match together: same region and page number under the wider of the two page masks, and equal ASIDs or either entry global. Invalid entries count. The removed entry reads back as an unwritten slot except for its PageMask: the kernel TLBRs its wired sentinel entries and reuses PageMask for the next write, and a zeroed mask made it write a per-CPU mapping as 4 KiB. The cost is confined to TLB writes on these models. An entry that overlaps the new one covers a lookup-map slot the new one covers, so a write whose slots are all empty or its own skips the scan. A PageMask that is not a run of ones, which software can write, defeats the map's ranges and turns the shortcut off. A refill that replaces a one-page entry nothing else covers skips the walk over the other entries. TLBWI and TLBWR are never compiled by jitv2, so both reach the same write and set TS in the interpreter. Tests: a 64-bit kernel missing on kuseg and xsseg on the R10000 and on the R4400, which chooses by mode; the Tezro sequence with the removed entries' PageMask kept; and TS from TLBWI and TLBWR. They fail with the rule off. --- src/cpu/mips_exec.rs | 27 ++++++++- src/cpu/mips_exec_test.rs | 49 +++++++++++++++ src/cpu/mips_tlb.rs | 124 +++++++++++++++++++++++++++++++++++++- src/cpu/mips_tlb_test.rs | 42 +++++++++++++ 4 files changed, 239 insertions(+), 3 deletions(-) diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 60204a0cb..d43be4755 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -2215,6 +2215,13 @@ impl MipsExecutor { self.sysad.describe_fault(phys_addr, size).map_or_else(String::new, |why| format!(" [{why}]")) } + /// Status.TS follows a TLB write that left two entries matching. Branchless: + /// a refill handler writes often and the flag is almost always clear. + #[inline(always)] + fn note_tlb_write_conflict(&mut self) { + self.core.cp0_status |= crate::cpu::mips_core::STATUS_TS & 0u32.wrapping_sub(self.tlb.take_write_conflict() as u32); + } + /// Create a new executor from a config and a bus (sysad) and a TLB. /// The cache hierarchy is constructed internally as a unified R4000Cache. pub fn new(sysad: Arc, mut tlb: T, cfg: &MipsCpuConfig) -> Self @@ -2226,6 +2233,7 @@ impl MipsExecutor { // and xkphys do. tlb.set_va_bits(C::VA_BITS); tlb.set_cache_attrs(Self::cache_attr_table()); + tlb.set_conflict_invalidation(C::R10K_FAMILY); core.entryhi_mask = model_entryhi_mask::(); if C::R10K_FAMILY { core.framemask_mask = 0xFFFF; @@ -4696,7 +4704,8 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", TlbResult::Miss { .. } => { if !DEBUG { self.update_tlb_exception_registers::(virt_addr); } // Miss: XTLB vector (0x080) for 64-bit extended addresses, UTLB vector (0x000) otherwise - TranslateResult::exc(if XTLB != 0 { exec_xtlb_miss(tlb_miss_code) } else { exec_tlb_miss(tlb_miss_code) }) + let xrefill = if C::R10K_FAMILY { self.r10k_xtlb_refill::(virt_addr) } else { XTLB != 0 }; + TranslateResult::exc(if xrefill { exec_xtlb_miss(tlb_miss_code) } else { exec_tlb_miss(tlb_miss_code) }) } TlbResult::Invalid { .. } => { if !DEBUG { self.update_tlb_exception_registers::(virt_addr); } @@ -4710,6 +4719,20 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } + /// Whether a refill miss on `virt_addr` takes the XTLB vector on the + /// R10000 family: it follows the address space (UX for user, SX for + /// xsseg, KX otherwise), not the mode, unlike the R4400 (R10000 UM v2.0 + /// pp. 333-334, 342). A 32-bit mode sign-extends the address first. + #[inline(always)] + fn r10k_xtlb_refill(&self, virt_addr: u64) -> bool { + use crate::cpu::mips_core::{STATUS_KX, STATUS_SX, STATUS_UX}; + const _: () = assert!(STATUS_SX == STATUS_UX << 1 && STATUS_KX == STATUS_SX << 1); + let va = if XTLB != 0 { virt_addr } else { virt_addr as u32 as i32 as i64 as u64 }; + // Region 0 is UX, 1 is SX, 2 and 3 are KX: three adjacent Status bits. + let bit = (va >> 62).min(2) as u32; + (self.core.cp0_status & (STATUS_UX << bit)) != 0 + } + /// Debug helper to translate address without side effects pub fn debug_translate(&mut self, virt_addr: u64) -> TranslateResult { self.translate_impl::(virt_addr, AccessType::Debug) @@ -7250,6 +7273,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let entry = self.create_tlb_entry_from_cp0(); self.ip28_trace_tlb_write("tlbwi", index, &entry); self.tlb.write(index, entry); + self.note_tlb_write_conflict(); // Flushes the nutlb too — the TLB it caches just changed. self.nanotlb_invalidate(); @@ -7279,6 +7303,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let entry = self.create_tlb_entry_from_cp0(); self.ip28_trace_tlb_write("tlbwr", index, &entry); self.tlb.write(index, entry); + self.note_tlb_write_conflict(); self.nanotlb_invalidate(); if mips_log(MIPS_LOG_TLB) { dlog_dev!(LogModule::Mips, "TLBWR: Write Random Index {}\n{}", index, self.tlb.format_entry(index)); } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index b7c4ff687..8564b42e9 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -1227,6 +1227,29 @@ mod tests { 0xAAAAAAAA means a stale nanotlb slot survived the switch"); } + /// A second write that would match the first removes it and sets Status.TS on + /// the R10000, for TLBWI and TLBWR; the R4400 keeps both. Fix: overlapping + /// writes (jitv2 never compiles them, so the interpreter is the only path). + #[test] + fn overlapping_tlb_writes_follow_the_cpu_family() { + use crate::cpu::mips_core::STATUS_TS; + use crate::cpu::mips_tlb::Tlb; + fn run>>(r10k: bool, funct: u32) { + let mut exec = tlb_exec::(Arc::new(MockMemory::new())); + let lo0 = (0x50 << 6) | (3 << 3) | (1 << 2) | (1 << 1); + tlb_op(&mut exec, &[(0, 5), (6, 7), (10, (0x100 << 13) | 10), (2, lo0)], FUNCT_TLBWI); + tlb_op(&mut exec, &[(0, 6)], funct); + assert_eq!(exec.core.cp0_status & STATUS_TS != 0, r10k, "funct {funct:#x}"); + assert_eq!(exec.tlb.read(5).entry_hi == 0, r10k, "funct {funct:#x}"); + } + on_big_stack(|| { + for funct in [FUNCT_TLBWI, FUNCT_TLBWR] { + run::(true, funct); + run::(false, funct); + } + }); + } + #[cfg(feature = "tlbcheck")] #[test] fn test_tlbwi_breaks_on_duplicate_entry() { @@ -5700,6 +5723,32 @@ mod tests { assert_ne!(st & EXEC_IS_XTLB_REFILL, 0, "true xkseg is unchanged: XTLB"); } + /// The R10000 picks the refill vector by the missing address's Status bit + /// (UX, SX or KX; R10000 UM v2.0 pp. 333-334), the R4400 by the current + /// mode: a 64-bit kernel missing on kuseg takes the 32-bit vector on the + /// R10000 only. Fix: refill vector. + #[test] + fn r10000_refill_vector_follows_the_address_space() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache as R10k; + use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_SX, STATUS_UX}; + use crate::cpu::mips_exec::EXEC_IS_XTLB_REFILL; + // A kernel-mode read of `va` with `xbits` in Status and an empty TLB: + // does it miss through the XTLB vector? + fn xrefill>>(xbits: u32, va: u64) -> bool { + let mut exec = tlb_exec::(Arc::new(MockMemory::new())); + exec.set_cp0_status((exec.core.cp0_status & !(STATUS_ERL | STATUS_EXL | STATUS_KX | 0x18)) | xbits); + let st = exec.read_data::<4>(va).expect_err("empty TLB must miss"); + assert_ne!(st & EXEC_IS_TLB_REFILL, 0); + st & EXEC_IS_XTLB_REFILL != 0 + } + on_big_stack(|| { + assert!(!xrefill::(STATUS_KX, 0), "kuseg follows UX"); + assert!(xrefill::(STATUS_UX, 0)); + assert!(xrefill::(STATUS_KX, 0), "R4400 follows the mode"); + assert!(!xrefill::(STATUS_KX, 1 << 62) && xrefill::(STATUS_KX | STATUS_SX, 1 << 62), "xsseg follows SX"); + }); + } + #[test] fn test_xtlb_compare_width_disambiguates_region_field() { std::thread::Builder::new() diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 275f513e0..166e2bc47 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -358,6 +358,17 @@ pub trait Tlb { fn set_cache_attrs(&mut self, _table: [CacheAttr; 8]) {} /// Set the implemented VPN width before installing entries. fn set_va_bits(&mut self, _bits: u32) {} + /// Make a TLB write invalidate every other entry it would match together + /// with, as the R10000 and R12000 do (NEC U10278EJ4V0UM, Status register + /// TS bit, p.223, and "Avoiding TLB Conflict", p.300): they do not shut + /// down on a multiple match. The R14000 and R16000 are given the same + /// rule as a model choice, matching QEMU's r10k_invalidate_conflicts(); + /// no manual for them was read. Off for the R4400 and R5000, whose + /// behaviour on a multiple match is not modelled. + fn set_conflict_invalidation(&mut self, _on: bool) {} + /// Whether a write since the last call invalidated another entry. The + /// executor reports it in Status.TS. + fn take_write_conflict(&mut self) -> bool { false } fn save_state(&self) -> toml::Value { toml::Value::Table(Default::default()) } fn load_state(&mut self, _v: &toml::Value) -> Result<(), String> { Ok(()) } @@ -560,9 +571,76 @@ pub struct MipsTlb { vmap_touched: std::collections::HashSet, #[cfg(feature = "tlbstats")] pub stats: TlbStats, + /// R10000 family: a write invalidates the entries it overlaps. See + /// [`Tlb::set_conflict_invalidation`]. + invalidate_on_write: bool, + /// Entries written since power-on; a never-written slot cannot conflict. + written: u64, + /// A write invalidated another entry since `take_write_conflict`. + write_conflict: bool, + /// Some entry written since power-on has a PageMask that is not a run of + /// ones from bit 13 (software can write any of bits 24:13). The map + /// ranges of such an entry are not exact, so `invalidate_overlapped` then + /// scans instead of asking the map. + irregular_mask: bool, } impl MipsTlb { + /// The R10000 family never lets a write create a multiple match: every + /// other entry the incoming one could match together with, wired or not, + /// is invalidated first and Status.TS records it (see + /// [`Tlb::set_conflict_invalidation`]). Write path only. + fn invalidate_overlapped(&mut self, index: usize, new: &TlbEntry) { + // An entry that overlaps the new one covers a lookup-map slot the new + // one covers (the map is keyed on VA[31:13], and agreeing on more bits + // agrees on these), and a slot covered by two entries is tagged + // VMAP_MULTI or names the other one. So when every slot of the new + // entry's range is empty or names only `index` (the entry being + // replaced), nothing overlaps and the scan can be skipped. + if !self.irregular_mask { + let (start, count) = Self::vmap_range(new); + let tag = index as u8; + // A 4 KiB page, the refill case, covers one slot. + let clear = if count == 1 { + let v = self.vmap[start]; + v == VMAP_MISS || v == tag + } else { + self.vmap[start..start + count].iter().all(|&v| v == VMAP_MISS || v == tag) + }; + if clear { + return; + } + } + let mut live = self.written & !(1u64 << index); + while live != 0 { + let i = live.trailing_zeros() as usize; + live &= live - 1; + if i >= self.num_entries { + continue; + } + let other = &self.entries[i]; + if !(new.is_global() || other.is_global() || new.asid() == other.asid()) { + continue; + } + let both = new.page_mask | other.page_mask | 0x1FFF; + if (new.entry_hi ^ other.entry_hi) & !both & self.vpn_mask64 != 0 { + continue; + } + let old_range = Self::vmap_range(other); + // TLBR of the entry reads zeros for EntryHi and EntryLo but still + // loads PageMask: software that sets PageMask once and relies on a + // read-back restoring it would otherwise write its next entry + // with a 4 KiB mask. + let mut gone = TlbEntry::new(); + gone.page_mask = other.page_mask; + self.entries[i] = gone; + self.shadow[i] = ShadowEntry::invalid(); + self.written &= !(1u64 << i); + self.vmap_vacate(i, old_range); + self.write_conflict = true; + } + } + pub fn new(num_entries: usize) -> Self { assert!(num_entries > 1 && num_entries <= TLB_MAX_ENTRIES, "MipsTlb supports 2..={} entries, got {}", TLB_MAX_ENTRIES, num_entries); @@ -579,6 +657,10 @@ impl MipsTlb { vmap_touched: std::collections::HashSet::new(), #[cfg(feature = "tlbstats")] stats: TlbStats::default(), + invalidate_on_write: false, + written: 0, + write_conflict: false, + irregular_mask: false, }; for list in 0..MRU_LISTS { tlb.mru_head[list] = 0; @@ -678,7 +760,26 @@ impl MipsTlb { /// step 3 that it now also aliases. #[inline] fn vmap_replace(&mut self, index: usize, old_range: (usize, usize)) { + self.vmap_vacate(index, old_range); + let (new_start, new_count) = Self::vmap_range(&self.entries[index]); + self.vmap_fill_range(index, new_start, new_count); + } + + /// Steps 1 to 3 of [`Self::vmap_replace`]: take `index` out of the map + /// and let the other entries reclaim what it covered. + #[inline] + fn vmap_vacate(&mut self, index: usize, old_range: (usize, usize)) { let (old_start, old_count) = old_range; + // One slot that only `index` covers (named by it, no VMAP_MULTI): + // no other entry covers it, so none has anything to reclaim and the + // walk below would find nothing. A refill replaces an entry this way. + // Slot 0 is left to the walk: the zeroed entries of a TLB that was + // never written, and the entries an overlapping write cleared, all + // read as covering it. + if old_count == 1 && old_start != 0 && self.vmap[old_start] == index as u8 { + self.vmap_clear(old_start, 1); + return; + } self.vmap_clear(old_start, old_count); let old_end = old_start + old_count; for i in 0..self.num_entries { @@ -694,8 +795,6 @@ impl MipsTlb { self.vmap_fill_range(i, lo, hi - lo); } } - let (new_start, new_count) = Self::vmap_range(&self.entries[index]); - self.vmap_fill_range(index, new_start, new_count); } /// Full internal consistency check (feature = "tlbcheck"). @@ -1082,6 +1181,11 @@ impl Tlb for MipsTlb { (entry.entry_lo[0] << 6) & pfn_mask, (entry.entry_lo[1] << 6) & pfn_mask, ]; + self.irregular_mask |= mask & (mask + 1) != 0; + if self.invalidate_on_write { + self.invalidate_overlapped(index, &entry); + } + self.written |= 1 << index; let old_range = Self::vmap_range(&self.entries[index]); self.entries[index] = entry; self.shadow[index] = ShadowEntry::from_entry(&entry, &self.c_attr); @@ -1203,6 +1307,14 @@ impl Tlb for MipsTlb { output } + fn set_conflict_invalidation(&mut self, on: bool) { + self.invalidate_on_write = on; + } + + fn take_write_conflict(&mut self) -> bool { + std::mem::take(&mut self.write_conflict) + } + fn set_va_bits(&mut self, bits: u32) { self.vpn_mask64 = 0xC000_0000_0000_0000 | vpn2_mask(bits); } @@ -1225,6 +1337,9 @@ impl Tlb for MipsTlb { self.mru_next[list][self.num_entries - 1] = MRU_NONE; } self.vmap.fill(VMAP_MISS); + self.written = 0; + self.write_conflict = false; + self.irregular_mask = false; #[cfg(feature = "tlbcheck")] { self.vmap_touched.clear(); } } @@ -1281,6 +1396,11 @@ impl Tlb for MipsTlb { self.mru_next[list][self.num_entries - 1] = MRU_NONE; } self.vmap.fill(VMAP_MISS); + self.written = !0u64 >> (64 - self.num_entries); + self.irregular_mask = self.entries[..self.num_entries].iter().any(|e| { + let mask = e.page_mask | 0x1FFF; + mask & (mask + 1) != 0 + }); for i in 0..self.num_entries { let (start, count) = Self::vmap_range(&self.entries[i]); self.vmap_fill_range(i, start, count); diff --git a/src/cpu/mips_tlb_test.rs b/src/cpu/mips_tlb_test.rs index 4ebbaeda1..be121f291 100644 --- a/src/cpu/mips_tlb_test.rs +++ b/src/cpu/mips_tlb_test.rs @@ -4,6 +4,48 @@ mod tests { use crate::cpu::mips_tlb::*; use crate::cpu::mips_exec::CacheAttr; + /// Region 3 base for the overlap test below. + const BASE: u64 = 0xc000_0000_0000_0000; + + fn entry(mask: u64, hi: u64, lo: [u64; 2]) -> TlbEntry { + let mut e = TlbEntry::new(); + (e.page_mask, e.entry_hi, e.entry_lo) = (mask, hi, lo); + e + } + + /// CPU 1 of a two-processor Tezro: the firmware flushes the TLB with 64 + /// invalid entries on the kernel's page number, then the kernel writes the + /// valid global 16 MiB entry over them. With the rule the valid entry + /// answers; without it an invalid duplicate does. A removed entry reads back + /// as unwritten with its PageMask. + /// Fix: overlapping writes. + #[test] + fn a_write_invalidates_the_entries_it_overlaps() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let va = BASE + 0x1234; + let build = |on: bool| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(on); + for index in (2..64).rev() { + tlb.write(index, entry(0x1fe000, BASE, [0; 2])); + } + tlb.write(1, entry(0x1ffe000, BASE | 0x1000, [(0x40 << 6) | (5 << 3) | 0x7; 2])); + // A later write elsewhere; unfixed, it left an invalid duplicate answering. + tlb.write(0, entry(0x6000, 0xc000_0fff_ffff_9000, [1, 0x2e42f])); + tlb + }; + assert!(matches!(build(false).translate::<1>(va, 0, AccessType::Fetch), TlbResult::Invalid { .. })); + let mut tlb = build(true); + assert!(tlb.take_write_conflict()); + match tlb.translate::<1>(va, 0, AccessType::Fetch) { + TlbResult::Hit { phys_addr, .. } => assert_eq!(phys_addr, 0x1234), + other => panic!("expected the valid entry to answer, got {other:?}"), + } + let gone = tlb.read(2); + assert_eq!((gone.entry_hi, gone.entry_lo, gone.page_mask), (0, [0; 2], 0x1fe000)); + }).unwrap().join().unwrap(); + } + #[test] fn test_tlb_entry_flags() { let mut entry = TlbEntry::new(); From 57a91335bd10c654ef5b64877dd5c2a86d07a80c Mon Sep 17 00:00:00 2001 From: James Braid Date: Sat, 31 Oct 2026 09:34:16 -0700 Subject: [PATCH 3/7] cpu: fix three R10000-family load/store differences Uncached stores nothing answers. A store that nothing answers raised a synchronous Data Bus Error with a precise PC on every CPU. The R10000 cannot do that. It sends an uncached store as a write request, and a write request gets no response; Bus Error comes only from an error response to a read or upgrade request, or from uncorrectable read data (R10000 UM pp. 104, 316). The processor never learns that a store failed. The R12000 manual documents the same no-response write path, and the later parts are taken to match, so the test is the R10000 family. CpuModel:: POSTS_UNCACHED_STORES, true for the family's caches, makes a refused store log and complete in write_data_impl and in the masked-store path (SWL, SWR, SDL, SDR), which the interpreter and the compiled code both go through. An uncached SC or SCD that is refused completes and succeeds (rt = 1); that is a model choice, since the manual says nothing about uncached store-conditional. Loads, instruction fetches, cached accesses and the R4400 and R5000 keep the exception, and device return values are untouched because BUS_ERR also tells non-CPU callers that a request was declined. The error goes no further: nothing in iris raises an interrupt or sets an error register for a store the Octane's bus refuses. The log line prints the first four refused stores and then each power of two, with the running count, so a probe loop cannot flood stderr. The Octane's ide diagnostics store 0x591204 and then 0x4 to physical 0x1e130400, where nothing answers; before, the first store took a Data Bus Error, now both print one line and the run carries on. O2 memory alias. A mapped page whose physical address has a bit set above bit 31 could reach the PCI window instead of RAM on the O2. The O2 aliases RAM at 0x8000_0000, and fold_phys tested that window on the full translated address before cutting it to the 32 bits the board decodes. IRIX keeps software bits in its page-table entries just above the PFN, so such an address missed the window, then lost the high bit in the mask and landed on the PCI range. Cut to the decoded bits first, then apply the alias, then cut its target. The extra AND runs only on an alias hit. Only the O2 has a physical alias. Partial LWR. In 64-bit mode a partial LWR (offset 0 to 2) sign-extended the merged word on every CPU. The manual calls the upper word of a partial LWR implementation specific (NEC U12754E pp. 125-126), and the reference results observed for the R10000 keep the old bits 63:32 at all three partial offsets; with sign extension a test that compares those results miscompares. The interpreter and the JIT now merge the old upper word on the R10000; a full word (offset 3) still sign-extends. The upper word is kept only when 64-bit operations are enabled for the current mode, matching QEMU; otherwise it sign-extends. The later parts are not covered, since nothing shows what they do. The choice is a JitConsts field set from CpuModel::PARTIAL_LWR_KEEPS_UPPER, so compile workers see it. Whether 64-bit operations are on is a byte in the core that the executor sets wherever it re-derives the privilege state, so a compiled LWR spends a byte load and a compare on it rather than decoding Status. The code-cache fingerprint gains lwr_upper=1 only when it is set, so the R4400 and R5000 machines keep their warm caches. The uncached-store and LWR changes affect the Octane, IP28 and an O2 with the R10000; the alias change affects the O2. The R4400 and R5000 are unchanged. Tests: SW, SWL and SC on the R10000 against a bus nothing answers, with the R5000 and a load still faulting; a page at 0x1_8000_0000 with the O2's mask and window expects RAM at 0; and LWR compiled and interpreted, at a partial and a full offset, with the mode's 64-bit bit set and clear, on the R10000 and the R5000, plus the byte the compiled LWR reads across Status writes and ERET. --- src/cpu/jitv2/codegen.rs | 39 +++++++++++++++++++++ src/cpu/jitv2/equiv_test.rs | 57 +++++++++++++++++++++++++++++++ src/cpu/jitv2/mod.rs | 1 + src/cpu/mips_cache_v2.rs | 12 +++++++ src/cpu/mips_core.rs | 2 ++ src/cpu/mips_exec.rs | 65 +++++++++++++++++++++++++++-------- src/cpu/mips_exec_test.rs | 67 +++++++++++++++++++++++++++++++++++++ 7 files changed, 229 insertions(+), 14 deletions(-) diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index 197d3c872..91f1a4b68 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -868,6 +868,10 @@ impl Codegen { for v in isa.isa_flags() { let _ = write!(s, " {v}"); } + // Only when set, so machines without the behaviour keep their warm cache. + if self.jit_consts.lwr_keeps_upper { + s.push_str(" lwr_upper=1"); + } s.push_str(" ops="); s.extend(crate::cpu::jitv2::opcode_support::enabled_snapshot().iter().map(|&b| if b { '1' } else { '0' })); Some(blake3::hash(s.as_bytes()).as_bytes()[..16].try_into().unwrap()) @@ -4403,6 +4407,11 @@ pub struct JitConsts { /// to loading from memory, exactly as before, so an un-stamped `Codegen` /// stays correct (just no faster). pub core: Option, + /// A partial LWR keeps bits 63:32 of the destination, as the R10000 + /// does (`CpuModel::PARTIAL_LWR_KEEPS_UPPER`). A model fact rather than a + /// core one, published beside the core address because both reach a worker's + /// `Codegen` the same way. The code-cache fingerprint carries it when set. + pub lwr_keeps_upper: bool, } impl JitConsts { @@ -10077,6 +10086,31 @@ fn emit_lwl_ldl(ctx: &mut EmitCtx, size: MemSize) { emit_write_gpr(ctx, field_rt(ctx.raw), result); } +/// LWR on a model whose partial LWR keeps rt[63:32] (`lwr_keeps_upper`), +/// mirroring `MipsExecutor::exec_lwr`: offsets 0 to 2 keep the upper word while +/// 64-bit operations are on, anything else sign-extends the merged low half. +/// `core.lwr_upper_below` makes that one compare. `loaded` is the aligned word. +fn emit_lwr_keeping_upper(ctx: &mut EmitCtx, loaded: Value, byte_offset: Value) { + let i64t = ir::types::I64; + // Bytes of the word the register keeps: 8 * (3 - offset). + let inverted = ctx.builder.ins().bxor_imm_s(byte_offset, 3); + let shift = ctx.builder.ins().ishl_imm_s(inverted, 3); + let ones = ctx.builder.ins().iconst(i64t, 0xFFFF_FFFF); + let mask = ctx.builder.ins().ushr(ones, shift); + let shifted_mem = ctx.builder.ins().ushr(loaded, shift); + let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw)); + let not_mask = ctx.builder.ins().bnot(mask); + let preserved = ctx.builder.ins().band(rt_val, not_mask); + let merged = ctx.builder.ins().bor(shifted_mem, preserved); + let low = ctx.builder.ins().ireduce(ir::types::I32, merged); + let sign_extended = ctx.builder.ins().sextend(i64t, low); + let below_off = ir::immediates::Offset32::new(std::mem::offset_of!(MipsCore, lwr_upper_below) as i32); + let below = ctx.builder.ins().uload8(i64t, MemFlagsData::trusted(), ctx.core_ptr, below_off); + let keep = ctx.builder.ins().icmp(IntCC::UnsignedLessThan, byte_offset, below); + let result = ctx.builder.ins().select(keep, merged, sign_extended); + emit_write_gpr(ctx, field_rt(ctx.raw), result); +} + /// LWR rt, imm(rs) / LDR rt, imm(rs): load the "right" (low-address-end) /// portion — the mirror-image shift direction of [`emit_lwl_ldl`], see that /// function's doc comment for the shared shape. Mirrors @@ -10094,6 +10128,11 @@ fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) { let loaded = emit_mem_read(ctx, aligned_addr, size); + if size == MemSize::B4 && ctx.jit_consts.lwr_keeps_upper { + emit_lwr_keeping_upper(ctx, loaded, byte_offset); + return; + } + let ity = size.ir_type(); let mem_val = if size == MemSize::B8 { loaded } else { ctx.builder.ins().ireduce(ity, loaded) }; let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw)); diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index a4587a0b2..8a24f3d76 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -4383,6 +4383,63 @@ mod tests { } } + /// One LWR at 0x1000 reading 0x2000 + `offset`, interpreted and compiled, on + /// CPU model `C` with CP0 Status `status`. The JIT learns the model's + /// choices from `install_jit_hooks`, as in production. + fn lwr_both(status: u32, offset: u64) -> (CoreSnapshot, CoreSnapshot) + where + C: crate::cpu::mips_cache_v2::CpuModel + From>, + { + let lwr = make_i(crate::cpu::mips_isa::OP_LWR, 1, 2, 0); + let build = || { + let mem = Arc::new(MockMemory::new_not_compilable()); + mem.set_word(0x2000, 0xDEAD_BEEF); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(mem, PassthroughTlb::default(), &cfg)); + (exec.core.gpr[1], exec.core.gpr[2], exec.core.pc) = (0x2000 + offset, 0x1122_3344_5566_7788, 0x1000); + exec.set_cp0_status(status); + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + exec + }; + let mut interp = build(); + interp.jitv2_dispatch_enabled = false; + interp.exec(lwr); + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[0] = lwr; + let mut analyzer = Analyzer::new(); + let mut instrs_owned = *analyzer.walk_bounded(&page, 0, 0x1000, 1).0; + let mut jit = build(); + jit.install_jit_hooks(); + // Workers read the model's choice from here; stamp the codegen from it. + let mut codegen = Codegen::new(); + codegen.jit_consts.lwr_keeps_upper = jit.jitv2.lock().jit_consts.lock().lwr_keeps_upper; + let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false).expect("compilable region"); + unsafe { jit_fn(&mut jit.core as *mut MipsCore) }; + std::mem::forget(codegen); + (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core)) + } + + /// Compiled and interpreted LWR agree: the R10000 keeps bits 63:32 of a + /// partial LWR while 64-bit operations are on for the mode; the R5000 and a + /// 32-bit user process (UX=0) sign-extend. Fix: partial LWR. + #[test] + fn jit_lwr_matches_the_interpreter_on_the_r10000_and_r5000() { + use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R5000ShadowCache}; + use crate::cpu::mips_core::STATUS_KX; + const USER: u32 = 2 << 3; + // (Status, 64-bit operations on): the mode's own bit decides. + for (status, ops64) in [(STATUS_KX, true), (USER | STATUS_KX, false)] { + let (interp, jit) = lwr_both::(status, 1); + assert_eq!(jit, interp, "R10000 status {status:#x}"); + assert_eq!(interp.gpr[2] >> 32 == 0x1122_3344, ops64, "R10000 status {status:#x}"); + } + let (interp, jit) = lwr_both::(STATUS_KX, 1); + assert_eq!(jit, interp); + assert_ne!(interp.gpr[2] >> 32, 0x1122_3344, "R5000 sign-extends"); + } + #[test] fn lwl_lwr_combine_to_a_full_unaligned_word_load_matches_interpreter() { // The canonical compiler-generated idiom for an unaligned word diff --git a/src/cpu/jitv2/mod.rs b/src/cpu/jitv2/mod.rs index 43e49a7b7..576246911 100644 --- a/src/cpu/jitv2/mod.rs +++ b/src/cpu/jitv2/mod.rs @@ -92,6 +92,7 @@ mod zz_corpus { // (baked immediate vs load) is identical to production. crate::cpu::jitv2::codegen::JitConsts { core: core::num::NonZeroUsize::new(core as *mut MipsCore as usize), + ..Default::default() } }; // Synthetic corpus-measurement geometry: 16 KiB direct-mapped L1-D diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index ec4ecaf8b..860883123 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -571,6 +571,18 @@ pub trait CpuModel: MipsCache { /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and /// R16000 are a model choice that gives them the same TLB. const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; + /// A partial LWR keeps bits 63:32 of the destination. Observed on the + /// R10000; whether the later parts do the same is not established, so they + /// keep the sign extension for now. + const PARTIAL_LWR_KEEPS_UPPER: bool = Self::R10K_FAMILY; + /// An uncached store the system refuses completes instead of raising a + /// Data Bus Error. The R10000 sends it as a write request, which gets no + /// response, and Bus Error comes only from an error response to a read or + /// upgrade request or from uncorrectable read data (R10000 UM pp. 104, 316). + /// The R12000 manual documents the same path and the later parts are taken + /// to match. An uncached SC succeeding is a model choice. Loads and fetches + /// still fault. + const POSTS_UNCACHED_STORES: bool = Self::R10K_FAMILY; } pub trait MipsCache: Send + Sync { diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 1643bf6d4..e1b3dbf71 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -901,6 +901,7 @@ pub struct MipsCore { /// expected value, and whether an SC is to the same word. pub ll_value: u64, pub ll_phys: u64, + pub lwr_upper_below: u8, // A partial LWR keeps the upper word at offsets below this: 3 in 64-bit mode on the R10000 family, else 0 // Execution state pub running: bool, @@ -1403,6 +1404,7 @@ impl MipsCore { cp0_badvaddr: 0, cp0_entryhi: 0, cp0_status: 0, + lwr_upper_below: 0, cp0_cause: 0, cp0_epc: 0, cp0_prid: 0, diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index d43be4755..8dfbfa7a7 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -1036,12 +1036,12 @@ pub struct MipsExecutor { pub batch_hook: Option) + Send>>, /// A window of physical addresses that reaches memory at another address, /// `(base, size, target)`, applied to mapped and XKPHYS translations - /// before [`Self::phys_mask`]. The O2 has a no-ECC alias of its memory at + /// after [`Self::phys_mask`]. The O2 has a no-ECC alias of its memory at /// 0x8000_0000 that would otherwise reach the PCI window the bus decodes /// there. None on the other machines. pub phys_alias: Option<(u64, u64, u64)>, /// The physical address bits the board decodes, applied to mapped, - /// XKPHYS and ERL-identity translations after [`Self::phys_alias`]. The + /// XKPHYS and ERL-identity translations before [`Self::phys_alias`]. The /// CPU drives 36 or 40 bits, but the Indy, Indigo2 and O2 decode 32 and /// alias the rest away, and IRIX relies on it: its page-table entries keep /// software bits (the PFN lock among them) just above the PFN, so they @@ -2222,6 +2222,16 @@ impl MipsExecutor { self.core.cp0_status |= crate::cpu::mips_core::STATUS_TS & 0u32.wrapping_sub(self.tlb.take_write_conflict() as u32); } + /// An uncached store the system refused, on a model that posts stores + /// (`CpuModel::POSTS_UNCACHED_STORES`): say so and let the store complete. + /// The caller returns `BUS_OK`. Always built, so a missing device or a + /// guest bug stays visible in every build. + #[cold] + fn post_refused_store(&mut self, virt_addr: u64, phys_addr: u64, size: usize, val: u64, status: u32) { + eprintln!("Unsupported uncached write{} (posted, completes): PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", + size * 8, self.core.pc, virt_addr, phys_addr, val, status, self.fault_note(phys_addr, size)); + } + /// Create a new executor from a config and a bus (sysad) and a TLB. /// The cache hierarchy is constructed internally as a unified R4000Cache. pub fn new(sysad: Arc, mut tlb: T, cfg: &MipsCpuConfig) -> Self @@ -2660,6 +2670,7 @@ impl MipsExecutor { // set) -- the property a persistent code cache needs. // Several CPUs share compiled code, so no one core's address. core: if self.core.smp || std::env::var_os("IRIS_JIT_PIC").is_some() { None } else { core::num::NonZeroUsize::new(core_addr) }, + lwr_keeps_upper: C::PARTIAL_LWR_KEEPS_UPPER, }; *j.jit_consts.lock() = consts; // The inline-compile path (`jitv2_compile_inline`) takes this @@ -3268,6 +3279,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", pub fn update_translate_fn(&mut self) { use crate::cpu::mips_core::PrivilegeMode; let is_64bit = self.core.is_64bit_mode(); + self.core.lwr_upper_below = if C::PARTIAL_LWR_KEEPS_UPPER && is_64bit { 3 } else { 0 }; let privilege = self.core.get_privilege_mode(); self.translate_fn = match (is_64bit, privilege) { (false, PrivilegeMode::Kernel) => translate_32_kernel::, @@ -4663,15 +4675,17 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } - /// A CPU physical address as the board decodes it: through - /// [`Self::phys_alias`] if it falls in it, then cut to [`Self::phys_mask`]. + /// A CPU physical address as the board decodes it: cut to + /// [`Self::phys_mask`], then through [`Self::phys_alias`] if it falls in it. #[inline(always)] fn fold_phys(&self, pa: u64) -> u64 { - let pa = match self.phys_alias { - Some((base, size, target)) if pa.wrapping_sub(base) < size => pa - base + target, + // Cut first: the PTE software bits above the PFN are not part of the + // address the alias window sees. Window and target lie within the mask. + let pa = pa & self.phys_mask; + match self.phys_alias { + Some((base, size, target)) if pa.wrapping_sub(base) < size => (pa - base + target) & self.phys_mask, _ => pa, - }; - pa & self.phys_mask + } } /// TLB translation. When `DEBUG` is true, CP0 side-effect registers are @@ -5148,9 +5162,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.sysad.write64(phys_addr, val) }; if ws != BUS_OK && ws != BUS_BUSY && !DEBUG { - eprintln!("Bus error on uncached write{}: PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", SIZE*8, self.core.pc, virt_addr, phys_addr, val, ws, self.fault_note(phys_addr, SIZE)); + if C::POSTS_UNCACHED_STORES { + self.post_refused_store(virt_addr, phys_addr, SIZE, val, ws); + BUS_OK + } else { + eprintln!("Bus error on uncached write{}: PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", SIZE*8, self.core.pc, virt_addr, phys_addr, val, ws, self.fault_note(phys_addr, SIZE)); + ws + } + } else { + ws } - ws }; // jitv2_lockstep's load/store verification (see MipsCore's // lockstep_mem field doc comment, and read_data_impl's matching @@ -5204,9 +5225,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } let ws = self.sysad.write64_masked(phys_addr, val, mask); if ws != BUS_OK && ws != BUS_BUSY && !DEBUG { - eprintln!("Bus error on uncached write64_masked: PC={:016x} VA={:016x} PA={:016x} val={:016x} mask={:016x} status={:08x}{}", self.core.pc, virt_addr, phys_addr, val, mask, ws, self.fault_note(phys_addr, 8)); + if C::POSTS_UNCACHED_STORES { + self.post_refused_store(virt_addr, phys_addr, mask.count_ones() as usize / 8, val, ws); + BUS_OK + } else { + eprintln!("Bus error on uncached write64_masked: PC={:016x} VA={:016x} PA={:016x} val={:016x} mask={:016x} status={:08x}{}", self.core.pc, virt_addr, phys_addr, val, mask, ws, self.fault_note(phys_addr, 8)); + ws + } + } else { + ws } - ws } } @@ -6424,8 +6452,17 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let mask = 0xFFFFFFFFu32 >> shift; let result = (mem_word >> shift) | (rt_val & !mask); - // Sign-extend to 64 bits - self.core.write_gpr(rt_reg, result as i32 as i64 as u64); + // The manual leaves the upper word of a partial LWR + // implementation specific (NEC U12754E pp. 125-126); the R10000 + // keeps bits 63:32. A full word, or any word with 64-bit + // operations off, sign-extends. `lwr_upper_below` folds both. + let value = if C::PARTIAL_LWR_KEEPS_UPPER && (byte_offset as u8) < self.core.lwr_upper_below { + (self.core.read_gpr(rt_reg) & 0xffff_ffff_0000_0000) | result as u64 + } else { + // Sign-extend to 64 bits + result as i32 as i64 as u64 + }; + self.core.write_gpr(rt_reg, value); self.handle_exec_complete() } Err(status) => self.finish_status(status) diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 8564b42e9..c23d015ef 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6616,4 +6616,71 @@ mod tests { assert_eq!(entry::(0xFFFF_C000), (0, 0x0300_4000, (lo & 0x3FFF_FFFF) | 0xC0)); }); } + + /// A bus where nothing answers: every read and write is refused. + struct RefusingBus; + impl BusDevice for RefusingBus {} + + /// A store the system refuses completes on the R10000, which sends it as a + /// write request that gets no response (R10000 UM pp. 104, 316); the R5000 + /// and a load keep the Data Bus Error. SWL takes the masked-store path. + /// Fix: refused uncached stores. + #[test] + fn refused_uncached_store_completes_only_on_the_r10000() { + fn run>>(op: u32) -> (u32, u64) { + let bus: Arc = Arc::new(RefusingBus); + let mut exec = MipsExecutor::::new(bus, PassthroughTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + exec.cache.set_llbit(true); + exec.core.write_gpr(4, 0xFFFF_FFFF_A100_0000); // kseg1: uncached + exec.core.write_gpr(6, 0x1234_5678); + (exec.exec(make_i(op, 4, 6, 4)), exec.core.read_gpr(6)) + } + use crate::traits::BUS_ERR; + type R10K = crate::cpu::mips_cache_v2::R10000Cache; + type R5K = crate::cpu::mips_cache_shadow::R5000ShadowCache; + for op in [OP_SW, OP_SWL] { + assert_eq!((run::(op).0, run::(op).0), (EXEC_COMPLETE, BUS_ERR), "op {op:#x}"); + } + assert_eq!(run::(OP_SC), (EXEC_COMPLETE, 1), "SC succeeds"); + assert_eq!(run::(OP_LW).0, BUS_ERR, "a refused load still faults"); + } + + /// The O2's RAM alias at 0x8000_0000 is matched on the 32 bits the board + /// decodes, so a PTE software bit above the PFN (PA 0x1_8000_0000) cannot + /// send a mapped page to the PCI window. Fix: O2 memory alias. + #[test] + fn phys_alias_is_matched_after_the_physical_mask() { + on_big_stack(|| { + let mut exec = tlb_exec::(Arc::new(MockMemory::new())); + exec.phys_mask = u32::MAX as u64; + exec.phys_alias = Some((0x8000_0000, 0x1000_0000, 0)); + let lo0 = (0x18_0000 << 6) | (3 << 3) | (1 << 2) | (1 << 1); + tlb_op(&mut exec, &[(0, 0), (5, 0), (10, 0x4000_0000), (2, lo0), (3, 0)], FUNCT_TLBWI); + assert_eq!(exec.debug_translate(0x4000_0000).phys, 0); + }); + } + + /// Compiled LWR reads `core.lwr_upper_below` rather than decoding Status, so + /// it must follow the mode through ERET, which changes it without a Status + /// write. Fix: partial LWR. + #[test] + fn lwr_upper_below_follows_the_mode_through_status_writes_and_eret() { + use crate::cpu::mips_core::{STATUS_EXL, STATUS_KX, STATUS_UX}; + fn check>>(keeps_upper: bool) { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.set_cp0_status(STATUS_KX); + assert_eq!(exec.core.lwr_upper_below, if keeps_upper { 3 } else { 0 }, "{} kernel KX", C::NAME); + exec.set_cp0_status((2 << 3) | STATUS_UX | STATUS_EXL); // EXL forces kernel mode, KX clear + assert_eq!(exec.core.lwr_upper_below, 0); + exec.core.cp0_epc = 0x1000; + assert_eq!(exec.exec(0x4200_0018), EXEC_COMPLETE, "ERET"); + assert_eq!(exec.core.lwr_upper_below, if keeps_upper { 3 } else { 0 }, "{} ERET to user, UX set", C::NAME); + } + check::(true); + check::(false); + } } From 9ca3babd48ac5f7d8f1e027138d78b4e0da78865 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:45:00 +0000 Subject: [PATCH 4/7] cpu: keep R10000 cache diagnostic state A PROM's cache walk reads and writes the caches with Index Load Data, Index Store Data and Index Store Tag. The R10000 models dropped or kept the wrong things. Primary-cache data. Only the secondary cache had a data array, so an Index Load Data on a primary cache always returned zero and an Index Store Data was dropped. The primary caches now have data arrays, and each store is masked to its diagnostic field (R10000 UM pp. 262, 271-272): 36 data bits and 1 parity bit for the instruction cache, 32 and 4 for the data cache, 64 and 10 for the secondary. An Index Store Data to the secondary cache writes a quadword, so the other doubleword becomes zero and takes the same check bits (p. 91). A mapped primary Index Load Data or Index Store Data still takes TLB Refill and Invalid, so CACHE now translates the operand for those two; the other index operations still do not. The secondary data slot is now the address >> 3 where it was >> 4, so the walk gets one slot per doubleword instead of two doublewords sharing one. Secondary tags. Index Store Tag stored all 64 bits of a secondary-cache tag, so a walking-1s test read back bits the R10000 defines as reserved, which the manual says return zero (R10000 UM p. 270, figure 11-31). MRU, the physical tag, state, virtual index and ECC are kept, which assembled is 0x8000_000f_ffff_cdff, and the rest dropped. The mask is CpuModel::SECONDARY_TAG_MASK, all ones except on the R10000 family. MRU stays because the PROM writes it as TagHi bit 31 and reads it back. The later parts share the tag layout (the R12000 manual agrees; the R14000 is taken to match), and a PROM's tag test on the R14000 expects the reserved bits to read as zero. Index Store Data is a diagnostic walk's operation, so it runs out of line in a cold function. Which encodings translate, store a tag, load a tag or load ECC is decided by CacheOp::decode and tested as a bit of a mask computed once per model (CpuModel::CACHE_OPS), not by a chain of compares: written inline the extra case grew cache_op and exec_cache in every CPU model and moved the interpreter's hot loops, which cost sys/llsc about 2% on a ship build. This affects the Octane, the IP28 and an O2 with an R10000. Tests: a primary round trip and the quadword store; an all-ones tag store on the R10000 reads back the mask with an R4400 control that keeps TagLo bits 12 and 13, and the walking-1s round trip expects the reserved bits as zero. --- src/cpu/mips_cache_shadow.rs | 156 ++++++++++++++++++++++++----------- src/cpu/mips_cache_v2.rs | 80 ++++++++++++++++++ src/cpu/mips_exec.rs | 31 +++---- 3 files changed, 199 insertions(+), 68 deletions(-) diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 1b89777aa..d79e8d410 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -23,7 +23,9 @@ //! to it and returns them unchanged, so a walking-1s test over tag or data //! SRAM passes without anyone having to know the hardware's field layout. //! Where the emulator *must* know the layout — to decide whether a line is -//! valid, say — that is a separate question from storing the bits. +//! valid, say — that is a separate question from storing the bits. The one +//! exception is the R10000's secondary tag, whose reserved bits read as zero +//! (`CpuModel::SECONDARY_TAG_MASK`). //! //! The shadow is deliberately not consulted by `read`/`write`/`fetch`. If it //! ever needs to be, this type is the wrong shape and should say so loudly @@ -90,16 +92,6 @@ const MRU_BIT: u64 = 1 << 63; /// R4400 and R5000 index differently — see `tag_slot`. const WAYS: usize = 2; -/// Bits a secondary-cache tag retains. -/// -/// All of them, now. A 36-bit mask was inferred here from the PROM's -/// walking-0s phase, which writes an all-ones TagLo and expects back -/// 0x0000000f_ffffcdfe — but that truncation comes from the tag being -/// assembled as `(TagHi << 32) | TagLo[31:0]`, not from the array dropping -/// bits. Once the executor carried TagHi properly the mask did nothing except -/// discard the MRU bit, which the PROM writes as TagHi[31] and reads back. -const L2_TAG_MASK: u64 = u64::MAX; - /// Is the shadow cache's operation trace on, at this level of detail? /// /// `log l2c mask op` traces the tag operations a diagnostic actually cares @@ -204,11 +196,11 @@ impl< lladdr: UnsafeCell::new(0), ecc_in: UnsafeCell::new(0), ecc_out: UnsafeCell::new(0), - ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, 0)), - dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, 0)), - // Only the R10000 secondary keeps a data shadow: it is the one - // the IP28 PROM walks with Index_Store_Data, and a 1 MB array is - // cheap once. The R4000 family has no index data ops. + // The R10000 keeps data arrays for all three caches, for the PROM's + // Index_Load_Data / Index_Store_Data walks. The R4000 family has no + // index data ops. + ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, if Self::R10K { IC_SIZE / 4 } else { 0 })), + dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, if Self::R10K { DC_SIZE / 4 } else { 0 })), l2: UnsafeCell::new(Shadow::new(Self::L2_LINES, if Self::R10K { L2_SIZE / 8 } else { 0 })), tc_base: UnsafeCell::new(std::ptr::null_mut()), tc_bitmap: UnsafeCell::new(0), @@ -297,17 +289,54 @@ impl< /// Data slot for an R10000 `Index_Load_Data` / `Index_Store_Data`. /// - /// Same shape: way in bit 0, the rest addressing the array. The PROM walks - /// it at `…00`, `…01`, `…10`, `…11`, `…20` — one doubleword per operation - /// with the way bit shifted in beneath it. - fn data_slot(&self, len: usize, virt_addr: u64) -> usize { + /// Same shape: way in bit 0, the rest addressing the array, with four-byte + /// primary and eight-byte secondary slots. This is a model choice, enough + /// for a data walk to read back what it wrote. + fn data_slot(&self, sel: u32, len: usize, virt_addr: u64) -> usize { if len == 0 { return 0; } let way = (virt_addr as usize) & (WAYS - 1); - let word = (virt_addr as usize) >> 4; + let shift = if matches!(sel, CACH_PI | CACH_PD) { 2 } else { 3 }; + let word = (virt_addr as usize) >> shift; (word * WAYS + way) % len } + + /// R10000 `Index_Store_Data`, kept out of line: it runs only in a PROM's + /// diagnostic walk, and inlining it grew `cache_op` and moved the code + /// laid out after it. + #[cold] + #[inline(never)] + fn index_store_data(&self, sel: u32, virt_addr: u64, phys_addr: u64) { + let s = self.shadow(sel); + if s.data.is_empty() { + return; + } + let slot = self.data_slot(sel, s.data.len(), virt_addr); + // Diagnostic field widths: R10000 UM pp. 262, 271-272. + let (data_mask, ecc_mask) = if sel == CACH_PI { + (0xf_ffff_ffff, 1) + } else if sel == CACH_PD { + (0xffff_ffff, 0xf) + } else { + (u64::MAX, 0x3ff) + }; + let ecc = unsafe { *self.ecc_in.get() } & ecc_mask; + s.data[slot] = phys_addr & data_mask; + s.ecc[slot] = ecc; + if matches!(sel, CACH_SI | CACH_SD) { + // Index Store Data (S) writes a quadword: the selected + // doubleword plus a zero partner, sharing the check bits + // (R10000 UM p. 91, figure 5-7). + let partner = slot ^ WAYS; + s.data[partner] = 0; + s.ecc[partner] = ecc; + } + if l2_op_log(CACHE_LOG_HIT) { + crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", + s.ecc[slot]); + } + } } impl< @@ -563,12 +592,16 @@ impl< } match op { - // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim: - // a tag test is a round trip, and round trips do not require - // knowing what the bits mean. + // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim + // unless the part sets a secondary mask: a tag test is a round trip, + // and round trips do not require knowing what the bits mean. C_IST => { let idx = self.tag_slot(sel, virt_addr); - let mask = if matches!(sel, CACH_SI | CACH_SD) { L2_TAG_MASK } else { u64::MAX }; + let mask = if matches!(sel, CACH_SI | CACH_SD) { + ::SECONDARY_TAG_MASK + } else { + u64::MAX + }; if l2_op_log(CACHE_LOG_OP) { crate::dlog!(LogModule::L2c, "shadow: -> IST va={virt_addr:#012x} idx={idx} stores {:#018x}", phys_addr & mask & !MRU_BIT); @@ -608,7 +641,7 @@ impl< C_R10K_CBARRIER if Self::R10K && sel == CACH_PI => 0, C_R10K_ILD if Self::R10K && matches!(sel, CACH_PI | CACH_PD | CACH_SD) => { let s = self.shadow(sel); - let slot = self.data_slot(s.data.len(), virt_addr); + let slot = self.data_slot(sel, s.data.len(), virt_addr); if s.data.is_empty() { 0 } else { @@ -620,17 +653,8 @@ impl< s.data[slot] } } - C_R10K_ISD if Self::R10K && matches!(sel, CACH_SI | CACH_SD) => { - let s = self.shadow(sel); - let slot = self.data_slot(s.data.len(), virt_addr); - if !s.data.is_empty() { - s.data[slot] = phys_addr; - s.ecc[slot] = unsafe { *self.ecc_in.get() }; - if l2_op_log(CACHE_LOG_HIT) { - crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", - s.ecc[slot]); - } - } + C_R10K_ISD if Self::R10K => { + self.index_store_data(sel, virt_addr, phys_addr); 0 } @@ -768,33 +792,40 @@ mod tests { } /// A tag test is a round trip, so the shadow must return exactly the bits - /// it was given — including any the real layout would not use. The cache + /// it was given, except the reserved ones the manual reads as zero. The cache /// model that preceded this one decoded TagLo into ptag/state/pidx fields /// and re-encoded on the way out, which silently dropped the low seven /// bits and every state code it did not recognise. A walking-1s test then /// failed on its very first bit. #[test] - fn tags_round_trip_every_bit() { + fn tags_round_trip_every_defined_bit() { let c = cache(); for bit in 0..64 { // 32 and 63 are MRU control/state, not tag storage. if bit == 32 || bit == 63 { continue; } let v = 1u64 << bit; c.cache_op(C_IST | CACH_SD, 0, v); + let kept = ::SECONDARY_TAG_MASK & v; assert_eq!( c.cache_op(C_ILT | CACH_SD, 0, 0), - v, + kept, "bit {bit} did not survive a store/load tag round trip" ); } } - /// Index_Store_Data / Index_Load_Data likewise. + /// Index_Store_Data / Index_Load_Data likewise, in the secondary and the + /// primary caches: a PROM's data walk round-trips. #[test] fn secondary_data_round_trips() { let c = cache(); c.cache_op(C_R10K_ISD | CACH_SD, 0x40, 0xdeadbeef); assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 0x40, 0), 0xdeadbeef); + + c.cache_op(C_R10K_ISD | CACH_PI, 4, 0x1234); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, 4, 0), 0x1234); + c.cache_op(C_R10K_ISD | CACH_PD, 8, 0x5678); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PD, 8, 0), 0x5678); } /// The MRU bit is **shared between the ways of a set**: written through @@ -862,23 +893,37 @@ mod tests { assert_eq!(c.cache_op_ecc(), 0x3fe, "way 1's check bits"); } + /// A secondary Index Store Data writes a quadword: the other doubleword + /// becomes zero and shares the ten check bits (R10000 UM p. 91). + #[test] + fn nec_layout_stores_a_quadword_with_ten_check_bits() { + let c = cache(); + c.set_cache_ecc(0); + c.cache_op(C_R10K_ISD | CACH_SD, 8, 0xdead_beef); + c.set_cache_ecc(0xffff_fc01); + c.cache_op(C_R10K_ISD | CACH_SD, 0, 0x1111_2222_3333_4444); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 8, 0), 0, "zero partner"); + assert_eq!(c.cache_op_ecc(), 0x001, "shared check bits"); + } + /// An untouched set reports no MRU, so an ordinary tag read is not /// contaminated by it. #[test] fn an_untouched_set_has_no_mru_bit() { let c = cache(); - c.cache_op(C_IST | CACH_SD, 0, 0xdead_beef); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0xdead_beef); + // Only defined tag bits (TagLo bits 9, 12 and 13 read as zero). + c.cache_op(C_IST | CACH_SD, 0, 0xdead_8cef); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0xdead_8cef); } /// Separate lines must not alias onto one another. #[test] fn distinct_lines_hold_distinct_tags() { let c = cache(); - c.cache_op(C_IST | CACH_SD, 0, 0x1111_1111); - c.cache_op(C_IST | CACH_SD, 128, 0x2222_2222); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0x1111_1111); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 128, 0), 0x2222_2222); + c.cache_op(C_IST | CACH_SD, 0, 0x1111_0111); + c.cache_op(C_IST | CACH_SD, 128, 0x2222_0022); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0x1111_0111); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 128, 0), 0x2222_0022); } /// Bit 0 of a CACHE index selects the **way**, and the two ways of a set @@ -906,14 +951,14 @@ mod tests { for set in 0..4u64 { for way in 0..2u64 { let va = set * 128 + way; - c.cache_op(C_IST | CACH_SD, va, 0x1000 + set * 16 + way); + c.cache_op(C_IST | CACH_SD, va, 0x4000 + set * 16 + way); } } for set in 0..4u64 { for way in 0..2u64 { let va = set * 128 + way; assert_eq!(c.cache_op(C_ILT | CACH_SD, va, 0), - 0x1000 + set * 16 + way, + 0x4000 + set * 16 + way, "set {set} way {way} aliased"); } } @@ -961,6 +1006,19 @@ mod tests { assert_eq!(::NAME, "shadow", "existing IP28 snapshots"); } + /// The R10000 reads the reserved bits of a secondary tag as zero (R10000 UM + /// p. 270); the R4400 stores the tag as written. + #[test] + fn the_r10000_masks_secondary_tags() { + fn stored>>(tag: u64) -> u64 { + let c = C::from(Arc::new(Memory::new(1)) as Arc); + c.cache_op(C_IST | CACH_SD, 0, tag); + c.cache_op(C_ILT | CACH_SD, 0, 0) + } + assert_eq!(stored::(u64::MAX), 0x8000_000f_ffff_cdff); + assert_eq!(stored::(0x3000) & 0x3000, 0x3000, "R4400 keeps TagLo bits 12 and 13"); + } + /// The O2 R10000 differs from the IP28's only in its secondary line size. #[test] fn the_o2_r10000_is_the_ip28_part_with_shorter_secondary_lines() { diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 860883123..448c47726 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -85,6 +85,77 @@ pub const C_R10K_CBARRIER: u32 = 5 << 2; pub const C_R10K_ILD: u32 = 6 << 2; pub const C_R10K_ISD: u32 = 7 << 2; +/// What one CACHE encoding (target in bits 1:0, operation in 4:2) does on a +/// model. +struct CacheOp { + /// The address is translated (a hit operation), rather than used as an + /// index. + translates: bool, + /// The operation stores TagHi:TagLo into the cache. + stores_tag: bool, + /// The operation loads TagHi:TagLo from the cache. + loads_tag: bool, + /// The operation loads CP0 ECC with the selected doubleword's check bits. + loads_ecc: bool, +} + +impl CacheOp { + const fn decode(r10k: bool, cache_op: u32) -> CacheOp { + let op = cache_op & 0x1C; + let sel = cache_op & 3; + // The R10000 index operations that name a cache entry rather than an + // address: they are not translated. The primary Index Load/Store Data + // operations select by virtual address, but a mapped one still takes + // TLB Refill and Invalid (VR10000 Instruction Set UM p. 15), so they + // are. + let primary_data = r10k + && (op == C_R10K_ILD || op == C_R10K_ISD) + && (sel == CACH_PI || sel == CACH_PD); + let index_op = r10k + && ((op == C_R10K_CBARRIER && sel == CACH_PI) + || (op == C_R10K_ILD && (sel == CACH_PI || sel == CACH_PD || sel == CACH_SD)) + || (op == C_R10K_ISD && (sel == CACH_SI || sel == CACH_SD))); + let hit_op = op == C_CDX || op == C_HINV || op == C_HWBINV || op == C_HWB || op == C_HSV; + CacheOp { + translates: primary_data || (!index_op && hit_op), + stores_tag: op == C_IST || ((index_op || primary_data) && op == C_R10K_ISD), + loads_tag: op == C_ILT || (index_op && op == C_R10K_ILD), + // Index_Load_Tag also loads CP0 ECC with the selected doubleword's + // check bits (R4000 UM A-42); the R10000 does this on + // Index_Load_Data. + loads_ecc: (r10k && op == C_R10K_ILD) || (cfg!(feature = "cache-ecc") && !r10k && op == C_ILT), + } + } +} + +/// Which CACHE encodings (bit n = `cache_op` n) do each thing on a model, so +/// `exec_cache` tests a bit of a compile-time constant instead of re-deriving +/// the decision with compares on every operation. +pub struct CacheOpMasks { + pub translates: u32, + pub stores_tag: u32, + pub loads_tag: u32, + pub loads_ecc: u32, +} + +impl CacheOpMasks { + const fn for_model(r10k: bool) -> CacheOpMasks { + let mut m = CacheOpMasks { translates: 0, stores_tag: 0, loads_tag: 0, loads_ecc: 0 }; + let mut cache_op = 0; + while cache_op < 32 { + let op = CacheOp::decode(r10k, cache_op); + let bit = 1 << cache_op; + if op.translates { m.translates |= bit; } + if op.stores_tag { m.stores_tag |= bit; } + if op.loads_tag { m.loads_tag |= bit; } + if op.loads_ecc { m.loads_ecc |= bit; } + cache_op += 1; + } + m + } +} + + /// Decode a raw cache_op field (5-bit: op[4:2] | target[1:0]) to a human-readable name. /// Matches the disassembler mnemonic convention used by gas/objdump. pub fn cache_op_name(op: u32) -> &'static str { @@ -565,6 +636,9 @@ pub trait CpuModel: MipsCache { /// and `Index_Load_Data` returns into it, neither of which is true of the /// R4000 hit operations that share those encodings. const R10K_CACHE_OPS: bool = false; + /// What each CACHE encoding does on this model, derived from + /// [`Self::R10K_CACHE_OPS`]. + const CACHE_OPS: CacheOpMasks = CacheOpMasks::for_model(Self::R10K_CACHE_OPS); /// The part is in the R10000 family, which shares one programming model: /// FrameMask, and 44 virtual and 40 physical address bits. /// Established for the R10000. The R12000 is inferred to match: its manual's @@ -583,6 +657,12 @@ pub trait CpuModel: MipsCache { /// to match. An uncached SC succeeding is a model choice. Loads and fetches /// still fault. const POSTS_UNCACHED_STORES: bool = Self::R10K_FAMILY; + /// Bits `Index_Store_Tag` keeps for the secondary cache: all of them, except + /// on the R10000 family, whose reserved tag bits read as zero (R10000 UM + /// p. 270, figure 11-31). Kept are MRU (bit 63), the physical tag (TagHi 3:0 + /// and TagLo 31:14), state, virtual index and ECC; TagLo bits 9, 12 and 13 + /// and TagHi 30:4 read as zero. + const SECONDARY_TAG_MASK: u64 = if Self::R10K_FAMILY { 0x8000_000f_ffff_cdff } else { u64::MAX }; } pub trait MipsCache: Send + Sync { diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 8dfbfa7a7..c93b93b81 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -6643,20 +6643,15 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let cache_op = d.rt as u32; // Encoded operation: cache_sel in bits[1:0], op in bits[4:2] let op = cache_op & 0x1C; - // Determine if this is a Hit operation that needs address translation - // On an R10000 the encodings for 5/6/7 are index operations, not the - // R4000 hit operations — translating them would fault on an index that - // is not a valid virtual address. See C_R10K_ISD in mips_cache_v2. + // Hit operations need address translation; index operations use the + // address as an index. On an R10000 the encodings for 5/6/7 are index + // operations, not the R4000 hit operations: translating them would + // fault on an index that is not a valid virtual address (see + // C_R10K_ISD in mips_cache_v2). `CpuModel::CACHE_OPS` holds that + // decision per encoding, so the choice here is one bit test. let sel = cache_op & 3; - let r10k_index_op = C::R10K_CACHE_OPS - && match op { - C_R10K_CBARRIER => sel == CACH_PI, - C_R10K_ILD => matches!(sel, CACH_PI | CACH_PD | CACH_SD), - C_R10K_ISD => matches!(sel, CACH_SI | CACH_SD), - _ => false, - }; - let needs_translation = - !r10k_index_op && matches!(op, C_CDX | C_HINV | C_HWBINV | C_HWB | C_HSV); + let bit = 1u32 << cache_op; + let needs_translation = C::CACHE_OPS.translates & bit != 0; let phys_addr = if needs_translation { // Hit operations need address translation @@ -6669,15 +6664,13 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", }; // For Index_Store_Tag, pass TagLo via phys_addr - let op = cache_op & 0x1C; // A cache tag is TagHi:TagLo, not TagLo alone. The IP28 PROM writes // the low 32 bits to $28 and the bits above to $29 — a 36-bit // secondary tag arrives as TagLo=0xffffcdfe, TagHi=0xf — so a model // that reads only TagLo sees a different tag from the one written. // Harmless for R4400/R5000, whose tags fit in 32 bits and who leave // TagHi zero: the shift-in contributes nothing there. - let stores_taglo = op == C_IST || (r10k_index_op && op == C_R10K_ISD); - let phys_addr_or_taglo = if stores_taglo { + let phys_addr_or_taglo = if C::CACHE_OPS.stores_tag & bit != 0 { ((self.core.cp0_taghi as u64) << 32) | (self.core.cp0_taglo as u32 as u64) } else { phys_addr @@ -6738,17 +6731,17 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } // Index_Load_Tag also loads CP0 ECC with the selected doubleword's // check bits (R4000 UM A-42); the R10000 does this on Index_Load_Data. - if (C::R10K_CACHE_OPS && op == C_R10K_ILD) - || (cfg!(feature = "cache-ecc") && !C::R10K_CACHE_OPS && op == C_ILT) { + if C::CACHE_OPS.loads_ecc & bit != 0 { self.core.cp0_ecc = self.cache.cache_op_ecc(); self.cache.set_cache_ecc(self.core.cp0_ecc); } // For Index_Load_Tag, update CP0 TagLo from result - if op == C_ILT || (r10k_index_op && op == C_R10K_ILD) { + if C::CACHE_OPS.loads_tag & bit != 0 { // Split back the way it arrived. Zeroing TagHi unconditionally // threw away the top of every tag wider than 32 bits. self.core.cp0_taglo = result & 0xFFFF_FFFF; + // A primary data cache access leaves TagHi zero (a model choice). self.core.cp0_taghi = (result >> 32) as u32; } From ef7e59c0e933e4d55caebc8a079ea21bff9bb432 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 6 Nov 2026 23:41:35 -0700 Subject: [PATCH 5/7] cpu: model the shipped processors as one chip table Add the R12000 and R14000, and replace the per-machine CPU constants with one table of the parts SGI shipped. The Octane shipped with an R12000 or R14000 and the O2 with an R12000, but `cpu` took only r4400, r5000 and r10000. An Octane hinv could not show the later parts, and CP0 Config reported 1 MB of secondary cache for every R10000-family part, so a 2 MB model printed "1 Mbyte". The new parts share the R10000's programming model (44-bit virtual, 40-bit physical, the R10000 cache operations) through model::r10k_family, and Config SS is now log2(secondary size / 512 KB). On the R12000 and later, Index Writeback Invalidate (D) also clears the line's primary data tag, as Index Load Tag then shows; the R10000 keeps it. They keep the upper word on a partial LWR as the R10000 does; for the R12000 and later that is a model choice, as is the R14000's TLB conflict rule. Each machine kept its own facts about the CPU: the Octane had three Count constants and a PRId, the O2 two Count rates, the IP28 another, and machine.rs, ip30_machine.rs and ip32_machine.rs each matched on the CPU type to pick a cache model. Now src/cpu/parts.rs holds one const per part: family, clock, revision and secondary cache size, as IRIX hinv reports them. The PRId is computed from the family and revision, Count is half the clock, and the name (r10000_195) is derived. Where two shipped parts share a family and clock, the earlier silicon takes an _EARLY suffix: the Indy and Indigo2 R5000 is rev 1.0 where the O2's is 2.1, and the IP28's R10000 is rev 2.5 where the Octane's is 2.7. Each machine lists the parts it shipped with in its own CPUS, first is the default: Indy, indigo2_ip22 r4400_150, r5000_180 (rev 1.0) indigo2_ip28 r10000_195 (rev 2.5) O2 r5000_180, r10000_250, r12000_400 Octane r10000_195, r12000_360, r14000_600 `[machine] cpu` stays the string the user wrote. It is resolved once, by MachineConfig::cpu_part, after the profile is final: no cpu is the machine's first part, a family name (`r14000`) or a part name (`r14000_600`) is the machine's part that matches, and anything else is refused by validate() with a message naming the parts it takes. The `[clock] fixed_mhz` override applies in ClockConfig::count_hz, and with_cpu! is the one place a part becomes a cache and executor type. The O2's R10000 and R12000 have 64-byte secondary lines (with 128 its PROM panics before its banner); the Octane R12000's 128 is a model choice. The R10000 Indigo2 is the IP28 board: Linux builds the IP22 sources for the R4X00 and R5000 and the IP28 sources only for the R10000, so an Indy or indigo2_ip22 no longer takes an R10000 and indigo2_ip28 takes nothing else. The GUI's Processor pickers read the same lists. The bare-metal benchmark suite runs on an Indy and measures every part there; it does not run on the IP28's memory map (RAM starts at 0x20000000, out of reach of the KSEG0 image). A config with the test device on therefore takes any part, so iris-bench keeps its r10000, r12000 and r14000 cells and a product config keeps the rule. Guest-visible changes: the Indy and Indigo2 R4400 is rev 5.0 at 150 MHz (Count 75 MHz; was rev 4.0 and 33) and their R5000 rev 1.0 at 180 MHz (was rev 2.1); the O2 R10000 is rev 3.4 at 250 MHz (was rev 2.5 at 180). The Octane and IP28 report what they did. validate() also refuses an O2 R4400, which only the constructor did. Tests: the PRId encoding, a family name picking a machine's first part of it, Config reporting each secondary cache, and Index Writeback Invalidate (D) on the R12000 with the R10000 as the control. --- CHANGELOG.md | 5 + FEATURES.md | 6 +- HACKING.md | 9 +- HELP.md | 12 +- bench/README.md | 4 + docs/indigo2-ip28.md | 1 + docs/o2-ip32.md | 16 +-- docs/octane-ip30.md | 8 +- iris-gui/src/bench_ui.rs | 8 +- iris-gui/src/config_ui.rs | 52 ++++----- iris-gui/src/dialogs/new_machine.rs | 28 ++--- iris-gui/src/main.rs | 27 +++-- src/bench_runner.rs | 27 +++-- src/bin/iris_bench.rs | 22 +++- src/config.rs | 167 ++++++++++++++-------------- src/cpu/build.rs | 28 +++++ src/cpu/mips_cache_shadow.rs | 62 +++++++++-- src/cpu/mips_cache_v2.rs | 11 +- src/cpu/mips_exec.rs | 32 +++--- src/cpu/mips_exec_test.rs | 17 +++ src/cpu/mips_tlb.rs | 2 +- src/cpu/mod.rs | 2 + src/cpu/parts.rs | 104 +++++++++++++++++ src/display_stop_tests.rs | 11 +- src/exit_persistence_tests.rs | 14 +-- src/ip30_machine.rs | 45 ++++---- src/ip32_machine.rs | 31 +++--- src/machine.rs | 70 +++++------- src/platform_profile_tests.rs | 4 - src/validate_config.rs | 10 +- 30 files changed, 531 insertions(+), 304 deletions(-) create mode 100644 src/cpu/build.rs create mode 100644 src/cpu/parts.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 291035328..3a91353a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ hashes are given where a change is easiest to understand by reading the commit. ## November 2026 +### R10000 family fixes, R12000 and R14000 + +- The R10000 family has 44-bit virtual and 40-bit physical addresses, FrameMask and a 28-bit EntryLo PFN, picks the TLB refill vector by address space, invalidates overlapped TLB writes (Status.TS), and completes uncached stores nothing answers. Partial LWR keeps the upper word, and the cache diagnostics (Index Store Data, secondary tag mask, Config size) work. +- `cpu` takes the parts each machine shipped with, such as `r12000_360` on the Octane, or a family name for the machine's own. The Indy and Indigo2 R4400 is now rev 5.0 at 150 MHz, the O2 R10000 rev 3.4 at 250 MHz, and an Indy no longer takes an R10000 (the R10000 Indigo2 is the IP28). The R12000 and R14000 are new. + ### Boot test before publishing - `build.yml` can boot IRIX 5.3 to a login prompt on Linux, macOS and Windows diff --git a/FEATURES.md b/FEATURES.md index c9174b22d..b254836a7 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -176,7 +176,7 @@ the first `[section]`, because TOML tables continue until the next header. | Key | Default | Values and behavior | |---|---|---| | `machine.profile` | `indy_ip24` | indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32 (configured by `[o2]`, see [the O2 guide](docs/o2-ip32.md)), octane_ip30 (configured by `[octane]`, see [the Octane guide](docs/octane-ip30.md)). | -| `machine.cpu` | `r4400` | r4400, r5000, r10000; CPU/ISA choice is independent of build features. An O2 takes r5000 or r10000 (an r4400 is refused when the machine is built); the Octane is r10000 only. | +| `machine.cpu` | the machine's first | A family (`r4400`, `r5000`, `r10000`, `r12000`, `r14000`) or a part (`r14000_600`); independent of build features. Indy and `indigo2_ip22`: r4400, r5000; `indigo2_ip28`: r10000; O2: r5000, r10000, r12000; Octane: r10000, r12000, r14000. The first listed is the default. | | `machine.cpus` | `1` | Processors fitted. 2 only on the Octane; see [its guide](docs/Octane/multiprocessor.md). | ### [graphics] @@ -297,7 +297,7 @@ Omit `[nfs]` to disable the share; the default forward list is empty. | Key | Default | Values and behavior | |---|---|---| -| `clock.fixed_mhz` | `33 on IP22/IP24; 97.5 on IP28; 90 on the O2 when unset` | CP0 Count MHz; IRIX's CPU inventory and the O2's PROM report twice this rate. | +| `clock.fixed_mhz` | `half the part's clock (75 on an R4400 Indy)` | CP0 Count MHz; IRIX's CPU inventory and the O2's PROM report twice this rate. | ### [rtc_offset] @@ -435,7 +435,7 @@ iris-gui does not parse this standalone option set; use its named-machine UI. | `--validate` | Check the configuration (parse, settings, files, directories, ports) without starting the machine; exit 0 or 1. See [HELP.md](HELP.md#checking-a-configuration). | | `--list-net-interfaces` | Print the host network interfaces libpcap can bridge onto, then exit. Requires a build with --features pcap. | | `--no-scsi-deferred-int` | Disable deferred SCSI status interrupts (default: enabled for OpenBSD/NetBSD compatibility). | -| `--cpu` | Emulated CPU: `r4400` (default), `r5000`, or `r10000`. Overrides `[machine] cpu`. | +| `--cpu` | Emulated CPU: a family (`r4400`, `r5000`, `r10000`, `r12000`, `r14000`) or a part (`r14000_600`) the machine takes. Overrides `[machine] cpu`. | | `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`), `v6`, `v8`, `v10`, `v12` (`vpro:v12` and so on, Octane only). | | `--raster` | VPro: `cpu` (default) or `wgpu`, the raster unit on the host GPU (`[graphics] raster`; needs a build with the `wgpu` feature). | | `--renderer` | What draws the window: `gl` (default) or `wgpu` (`renderer`; needs a build with the `wgpu` feature). | diff --git a/HACKING.md b/HACKING.md index dfb584948..ef3611725 100644 --- a/HACKING.md +++ b/HACKING.md @@ -18,7 +18,8 @@ Other documents: IRIS emulates five SGI workstations: the Indy (IP24), the Indigo2 (IP22), the Indigo2 IMPACT (IP28), the O2 (IP32) and the Octane (IP30). The CPU is an -R4400, R5000 or R10000, chosen at runtime (the Octane takes the R10000 only). +R4400, R5000, R10000, R12000 or R14000, chosen at runtime from the parts the +machine shipped with (`src/cpu/parts.rs`). Graphics is Newport (REX3), GR2 (XZ/Extreme) or IMPACT on the Indy and Indigo2, GBE on the O2 and Solid IMPACT on the Octane. It boots the real PROMs and IRIX 5.3 and 6.5 to a desktop with networking. @@ -271,8 +272,8 @@ in [docs/interrupt_map.md](docs/interrupt_map.md). ### Time -CP0 Count is derived from host wall-clock time at a fixed rate (33 MHz on -IP22/IP24, 97.5 MHz on IP28; `[clock] fixed_mhz` overrides). The Compare +CP0 Count is derived from host wall-clock time at a fixed rate (half the part's +clock, `Part::count_hz`: 75 MHz on the Indy R4400, 97.5 MHz on IP28; `[clock] fixed_mhz` overrides). The Compare interrupt and other periodic device events come from host timers (`src/hptimer.rs`), not from counting instructions. A guest that runs faster or slower than real hardware sees the same time passing. `--features @@ -371,7 +372,7 @@ consumer thread never takes `rec`. handler sets the PC itself, including delay slots. - The CPU model is the `CpuModel` trait (`MIPS4`, `PRID`, `FIR`, `TLB_ENTRIES`, ...), implemented by each cache model. The executor is - monomorphised over it, so all three CPUs are in every binary with no + monomorphised over it, so every CPU is in every binary with no per-model branch on the hot path. `Machine::new` picks one from `[machine] cpu`. - With `opcodefusion` (implied by `lightning`), the interpreter executes diff --git a/HELP.md b/HELP.md index ad825df4e..b162ac278 100644 --- a/HELP.md +++ b/HELP.md @@ -39,8 +39,8 @@ host. `lightning` builds and the JIT stack (`--features jitv2,gpujit`) give higher MIPS; the hinv MHz string stays the same. The guest's MHz comes from the CP0 Count rate, which is fixed: Count ticks at -33 MHz on IP22/IP24 (66 MHz CPU inventory) or 97.5 MHz on IP28 -(195 MHz CPU inventory), anchored to host wall-clock time. There is +half the configured part's clock (75 MHz for the 150 MHz R4400 of an Indy or +Indigo2, 97.5 MHz for the 195 MHz R10000 of an IP28), anchored to host wall-clock time. There is no calibration or inference. Override it with `[clock] fixed_mhz` or `--clock-fixed-mhz` if a guest needs something else. @@ -630,7 +630,7 @@ scsi_deferred_int = true # needed by OpenBSD/NetBSD; see --no-scsi-deferr [machine] profile = "indy_ip24" # "indigo2_ip22" (--ip22), "indigo2_ip28", or "o2_ip32" (an [o2] section: docs/o2-ip32.md); --machine NAME picks any -cpu = "r4400" # "r5000" or "r10000" (--cpu); use R10000 on IP28 +cpu = "r4400" # "r5000" (--cpu); IP28: "r10000", Octane: r10000/r12000/r14000, O2: r5000/r10000/r12000; or a part, "r14000_600" # cpus = 2 # Octane only: a second processor (docs/Octane/multiprocessor.md) [graphics] @@ -643,7 +643,7 @@ resolution = "guest" # or "1024x768", "1280x960", "1280x1024" # gpu_backend = "auto" # with raster = "wgpu": "vulkan", "metal", "dx12" or "gl" [clock] -# fixed_mhz = 33 # Count MHz: default 33 on IP22/IP24, 97.5 on IP28 +# fixed_mhz = 33 # Count MHz: default half the part's clock: 75 on the Indy and Indigo2 R4400, 97.5 on IP28 # [rtc_offset] # guest clock start, relative to host time (all signed) # years = -18 # years/months step the calendar (Mar 31 -1 month = Feb 28/29) @@ -801,9 +801,9 @@ Configuration --machine Machine to emulate (a [machine] profile name); `help` lists them --ip22 Same as --machine indigo2_ip22 --graphics newport, xz, extreme, solidimpact, highimpact, maximpact, v6, v8, v10, v12 - --cpu r4400 (default), r5000, or r10000 + --cpu r4400 (default), r5000, r10000, r12000 or r14000, as the machine allows --bank0..--bank3 RAM bank sizes (0/8/16/32/64/128; IP28: also 256/512) - --clock-fixed-mhz CP0 Count frequency (IP22/IP24: 33; IP28: 97.5) + --clock-fixed-mhz CP0 Count frequency (default: half the part's clock, 75 for an Indy R4400) Storage --scsi1/2/3/7 Hard disk image at that SCSI ID diff --git a/bench/README.md b/bench/README.md index b842e5394..a29af9307 100644 --- a/bench/README.md +++ b/bench/README.md @@ -340,6 +340,10 @@ and the normal mode for speed. The result is labelled | `r5000-jitv2` | `jitv2` | `r5000` | | `r4400-lightning` | `lightning` | `r4400` | | `r4400-jitv2-lightning` | `jitv2,lightning` | `r4400` | +| `r10000-interp`, `r12000-interp`, `r14000-interp` | (default) | `r10000_195`, `r12000_360`, `r14000_600` | + +The R10000 family always runs the shadow caches. The suite's Indy has the test +device on, which takes any part; a product Indy takes only the R4400 and R5000. Each build is copied to `bench/build/iris-` before the next one starts — the next `cargo build` overwrites `target/release/iris`, and a matrix that races diff --git a/docs/indigo2-ip28.md b/docs/indigo2-ip28.md index ad32257e2..dd38e0348 100644 --- a/docs/indigo2-ip28.md +++ b/docs/indigo2-ip28.md @@ -2,6 +2,7 @@ IP28 pairs the Indigo2 fullhouse devices with an R10000 CPU and IMPACT graphics. The same binary supports IP24, IP22, and IP28; no `ip28` Cargo feature is needed. +The profile takes only the R10000, and the Indy and IP22 profiles only the R4400 or R5000. IRIX 6.5 desktop boot and a two-bank 1 GB configuration have been recorded in the repository's validation notes. Device and graphics coverage is still evolving. diff --git a/docs/o2-ip32.md b/docs/o2-ip32.md index a1ee2daf5..4a8885f57 100644 --- a/docs/o2-ip32.md +++ b/docs/o2-ip32.md @@ -3,8 +3,8 @@ The O2 is SGI's unified-memory workstation: CRIME (the memory controller and rendering engine), MACE (the I/O chip: PCI, Ethernet, serial ports, PS/2, audio, the clocks) and GBE (the display engine, which scans the frame buffer -out of main memory). IRIS emulates it with an R10000 or an R5000, chosen at run -time like the other machines. +out of main memory). IRIS emulates it with an R5000, R10000 or R12000, chosen at +run time like the other machines. It runs IRIX 6.5 (tested with 6.5.22) to the desktop, with networking, audio, SCSI disks and CD-ROMs, X and OpenGL through the CRIME rendering engine, and @@ -28,7 +28,7 @@ serial_log = "serial.log" [machine] profile = "o2_ip32" -cpu = "r10000" # or "r5000" +cpu = "r10000" # or "r5000" or "r12000" [o2] control_port = 2293 @@ -65,14 +65,16 @@ in the file to silence the note. ## CPU, memory and clock -- **R10000 and R5000** both run the shadow cache model that IP28 uses, from +- **R5000, R10000 and R12000** all run the shadow cache model that IP28 uses, from reset: loads and stores go straight to memory. The PROM tests physical page 0 uncached while its stack sits in cache tags for that same page, so while those tags stand the bus answers for the page from a separate 4 KB page. - Linux supports only R5000-class O2s. -- `[clock] fixed_mhz` is CP0 Count, 90 MHz unless set; the PROM measures and - reports twice it as the CPU clock. R10000 O2s shipped at 175 and 195 MHz - (87.5 and 97.5), R5000s at 180 to 300. +- `[clock] fixed_mhz` is CP0 Count, half the part's clock unless set (90 MHz for + the R5000); the PROM measures and reports twice it as the CPU clock. `r10000` + is the rev 3.4 250 MHz part and `r12000` the rev 3.5 400 MHz part with a 2 MB + secondary cache. Both have 64-byte secondary lines, because the PROM fails with + 128-byte ones. - `banks` is the kit in each of the four DIMM pairs from A: 32, 64, 128 or 256 MB each, largest first, up to 1 GB. The default `[128, 128]` is 256 MB. IRIX boots with any of them, from a single 32 MB kit up. NetBSD's 32-bit diff --git a/docs/octane-ip30.md b/docs/octane-ip30.md index cc652a8e2..0798d29ee 100644 --- a/docs/octane-ip30.md +++ b/docs/octane-ip30.md @@ -59,9 +59,11 @@ overlay = true ``` Paths are relative to the directory iris runs in. Every key is listed in -[FEATURES.md](../FEATURES.md). `machine.cpu` must be `r10000`. `[clock] -fixed_mhz` sets CP0 Count; unset it is 97.5 MHz, half of the R10000's -195 MHz pipeline clock. +[FEATURES.md](../FEATURES.md). `machine.cpu` is `r10000` (revision 2.7), +`r12000` (revision 3.5, 2 MB secondary cache) or `r14000` (revision 2.4, 2 MB). +`[clock] fixed_mhz` sets CP0 Count, half the pipeline clock. Unset it is +97.5 MHz for the 195 MHz R10000, 180 MHz for a 360 MHz R12000 and 300 MHz for a +600 MHz R14000. ## PROM and flash diff --git a/iris-gui/src/bench_ui.rs b/iris-gui/src/bench_ui.rs index 9f0d7de51..c1490088b 100644 --- a/iris-gui/src/bench_ui.rs +++ b/iris-gui/src/bench_ui.rs @@ -44,7 +44,7 @@ pub struct BenchState { last: Option, String>>, quick: bool, /// Which CPU to benchmark. Runtime config, so this needs no separate build. - cpu: iris::config::CpuModel, + cpu: &'static iris::cpu::parts::Part, /// Parsed once. Normally empty — see `iris::bench_report::bundled_reference`. reference: Option, dev: DevRunner, @@ -62,7 +62,7 @@ impl Default for BenchState { // number, and is required before a result can go in the reference // table. quick: true, - cpu: iris::config::CpuModel::default(), + cpu: &iris::cpu::parts::R4400_150, reference: None, dev: DevRunner::default(), } @@ -102,7 +102,7 @@ impl BenchState { self.live.is_some() || self.dev.is_running() } - fn start(&mut self, quick: bool, cpu: iris::config::CpuModel) { + fn start(&mut self, quick: bool, cpu: &'static iris::cpu::parts::Part) { if self.is_running() { return; } let lines = Arc::new(Mutex::new(Vec::new())); @@ -240,7 +240,7 @@ fn controls(ui: &mut Ui, st: &mut BenchState, machine_running: bool) { egui::ComboBox::from_id_salt("bench_cpu") .selected_text(st.cpu.label()) .show_ui(ui, |ui| { - for m in iris::config::CpuModel::ALL { + for &m in iris::bench_runner::BENCH_CPUS { ui.selectable_value(&mut st.cpu, m, m.label()); } }) diff --git a/iris-gui/src/config_ui.rs b/iris-gui/src/config_ui.rs index 891d37d2e..5743d842a 100644 --- a/iris-gui/src/config_ui.rs +++ b/iris-gui/src/config_ui.rs @@ -1,8 +1,9 @@ use egui::{Color32, ComboBox, DragValue, Grid, RichText, ScrollArea, TextEdit, Ui}; use iris::build_features; +use iris::cpu::parts::Part; use std::path::Path; use iris::config::{AudioHost, CacheModel, - format_unix_utc, CpuModel, ForwardBind, ForwardProto, GraphicsBoard, KeyboardLayout, + format_unix_utc, ForwardBind, ForwardProto, GraphicsBoard, KeyboardLayout, MachineConfig, MachineProfile, NetMode, NfsConfig, PortForwardConfig, RtcOffset, ScsiDeviceConfig, ScsiKind, VinoSource, VinoStandard, VALID_BANK_SIZES, }; @@ -19,7 +20,7 @@ pub struct MemoryUiContext { pub started_banks: Option<[u32; iris::config::MAX_BANKS]>, /// The CPU the running guest actually booted on, so a pending change is /// shown as pending rather than as fact. `None` when nothing is running. - pub started_cpu: Option, + pub started_cpu: Option<&'static Part>, } /// A host network interface candidate for the PCAP backend selector. This is a @@ -284,27 +285,30 @@ fn show_general(ui: &mut Ui, cfg: &mut MachineConfig, mem_ctx: MemoryUiContext) // Only the CPUs this machine can be built with: the Octane has one, // the 195 MHz R10000, and an O2 refuses an R4400. ui.add_enabled_ui(!mem_ctx.running, |ui| { + let mut cpu = cfg.cpu(); ComboBox::from_id_salt("cpu_model") - .selected_text(cfg.machine.cpu.label()) + .selected_text(cpu.label()) .show_ui(ui, |ui| { for &c in cfg.machine.profile.cpus() { - ui.selectable_value(&mut cfg.machine.cpu, c, c.label()); + ui.selectable_value(&mut cpu, c, c.label()); } }); + if cpu != cfg.cpu() { cfg.set_cpu(cpu); } }); ui.end_row(); // `[machine] cache`. The R10000 only has the shadow model, so the // choice is fixed there. ui.label(t!("config-cache-model")); - let r10k = cfg.machine.cpu == CpuModel::R10000; + let cpu = cfg.cpu(); + let r10k = cpu.is_r10000_family(); ui.add_enabled_ui(!mem_ctx.running && !r10k, |ui| { - let shown = if r10k { t!("config-cache-shadow-only") } else { cache_model_label(cfg.machine.cache, cfg.machine.cpu) }; + let shown = if r10k { t!("config-cache-shadow-only") } else { cache_model_label(cfg.machine.cache, cpu) }; ComboBox::from_id_salt("cache_model") .selected_text(shown) .show_ui(ui, |ui| { for m in [CacheModel::Auto, CacheModel::Full, CacheModel::Shadow] { - ui.selectable_value(&mut cfg.machine.cache, m, cache_model_label(m, cfg.machine.cpu)); + ui.selectable_value(&mut cfg.machine.cache, m, cache_model_label(m, cpu)); } }) .response @@ -314,16 +318,8 @@ fn show_general(ui: &mut Ui, cfg: &mut MachineConfig, mem_ctx: MemoryUiContext) ui.label(t!("config-clock")); ui.horizontal(|ui| { - // CP0 Count: half the CPU clock on the R10000 machines (the Octane's - // is 195 MHz) and on the O2 (90 MHz); a third of it on the Indy and - // the Indigo2 (33 MHz). - let auto_default = if cfg.machine.profile.ip28() || octane { - 97.5 - } else if o2 { - 90.0 - } else { - 33.0 - }; + // CP0 Count: half the chip's clock. + let auto_default = cfg.cpu().count_hz() as f64 / 1e6; let mut mhz = cfg.clock.fixed_mhz.unwrap_or(auto_default); let changed = ui.add(DragValue::new(&mut mhz).range(0.1..=1000.0).suffix(" MHz")).changed(); if changed { cfg.clock.fixed_mhz = Some(mhz); } @@ -343,15 +339,15 @@ fn show_general(ui: &mut Ui, cfg: &mut MachineConfig, mem_ctx: MemoryUiContext) .color(Color32::from_rgb(220, 170, 90)), ); if let Some(started) = mem_ctx.started_cpu { - if started != cfg.machine.cpu { - ui.label(t!("config-pending", running = started.label(), pending = cfg.machine.cpu.label())); + if started != cfg.cpu() { + ui.label(t!("config-pending", running = started.label(), pending = cfg.cpu().label())); } } } else { subtitle(ui, t!("config-cpu-applies-subtitle")); } subtitle(ui, t!("config-cpu-subtitle")); - if cfg.machine.cpu == CpuModel::R10000 { + if cfg.cpu().is_r10000_family() { subtitle(ui, t!("config-r10000-subtitle")); } // jitv2's compile pool and its persistent code cache live on the Performance @@ -528,7 +524,7 @@ fn port_row(ui: &mut Ui, label: String, hover: String, value: &mut Option, } /// A cache model's menu label; Auto says what it resolves to in this build. -fn cache_model_label(m: CacheModel, cpu: CpuModel) -> String { +fn cache_model_label(m: CacheModel, cpu: &Part) -> String { match m { CacheModel::Auto if m.is_shadow(cpu) => t!("config-cache-auto-shadow"), CacheModel::Auto => t!("config-cache-auto-full"), @@ -679,12 +675,12 @@ fn show_octane_graphics_picker(ui: &mut Ui, cfg: &mut MachineConfig, running: bo /// DIMM-pair kits of 32/64/128/256 MB filled from pair A, so a config that was /// an Indy a moment ago carries both. fn o2_defaults(cfg: &mut MachineConfig) { - if !cfg.machine.profile.cpus().contains(&cfg.machine.cpu) { - cfg.machine.cpu = CpuModel::R5000; + if cfg.cpu_part().is_err() { + cfg.machine.cpu = None; } - // CP0 Count is half the CPU clock here (90 MHz), and a clock carried over - // from the Indy (33 MHz) would make IRIX's clock run at the wrong rate. - // Unset it so the O2's own default applies. + // CP0 Count is half the chip's clock, and a clock carried over from + // another machine would make IRIX's clock run at the wrong rate. Unset it + // so the chip's own default applies. cfg.clock.fixed_mhz = None; // Kits only, filled from pair A. Judge the layout by the core's own rules // rather than restating them here, so the two cannot drift. @@ -705,7 +701,7 @@ fn o2_defaults(cfg: &mut MachineConfig) { } fn octane_defaults(cfg: &mut MachineConfig) { - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = None; let sizes = MachineProfile::OctaneIp30.bank_sizes(); let banks = &cfg.banks[..MachineProfile::OctaneIp30.memory_banks()]; let filled = banks.iter().take_while(|&&b| b != 0).count(); @@ -1890,7 +1886,7 @@ fn show_debug(ui: &mut Ui, cfg: &mut MachineConfig) -> ConfigAction { // What this machine is configured to be, not how the binary was built — // all CPU models are always compiled in and selected at start. ui.label(t!("config-build-cpu")); - ui.label(cfg.machine.cpu.label()); + ui.label(cfg.cpu().label()); ui.end_row(); // `gpujit` = all three graphics JITs. The App Store build turns them // off at run time with IRIS_NO_JIT, so report what will actually run. diff --git a/iris-gui/src/dialogs/new_machine.rs b/iris-gui/src/dialogs/new_machine.rs index e2877b322..544b8b3e0 100644 --- a/iris-gui/src/dialogs/new_machine.rs +++ b/iris-gui/src/dialogs/new_machine.rs @@ -1,5 +1,6 @@ use eframe::egui::{self, Color32, ComboBox, Grid, RichText, TextEdit}; -use iris::config::{CpuModel, MachineConfig, MachineProfile, ScsiDeviceConfig, VALID_BANK_SIZES}; +use iris::config::{MachineConfig, MachineProfile, ScsiDeviceConfig, VALID_BANK_SIZES}; +use iris::cpu::parts::{self, Part}; use iris::dev::ng1::vc2_timings::NewportResolution; use crate::i18n::{subtitle, ResponseExt}; @@ -12,7 +13,7 @@ pub struct NewMachineDialog { pub name: String, /// Emulated SGI machine model (Indy IP24 / Indigo2 IP22/IP28). pub profile: MachineProfile, - pub cpu: CpuModel, + pub cpu: &'static Part, /// Host-forced Newport video mode (`Guest` leaves it to IRIX/setmon). pub resolution: NewportResolution, pub prom_path: String, @@ -41,7 +42,7 @@ impl Default for NewMachineDialog { open: false, name: "indy".into(), profile: MachineProfile::default(), - cpu: CpuModel::default(), + cpu: &parts::R4400_150, resolution: NewportResolution::default(), prom_path: "prom.bin".into(), use_embedded_prom: true, @@ -124,13 +125,13 @@ impl NewMachineDialog { // nudge the Processor pick to match; leaving it un-does the // nudge rather than stranding the user on R10000 elsewhere. if self.profile == MachineProfile::Indigo2Ip28 { - self.cpu = CpuModel::R10000; + self.cpu = MachineProfile::Indigo2Ip28.cpus()[0]; // Newport resolution presets don't apply to IMPACT. self.resolution = NewportResolution::Guest; } else if self.profile == MachineProfile::OctaneIp30 { // An R10000 machine with no built-in PROM; its RAM // is matched pairs of 64 to 512 MB in bank 0 up. - self.cpu = CpuModel::R10000; + self.cpu = MachineProfile::OctaneIp30.cpus()[0]; self.use_embedded_prom = false; self.ram_total_mb = 128; self.ram_banks = [128, 0, 0, 0]; @@ -140,14 +141,14 @@ impl NewMachineDialog { // RAM is DIMM-pair kits of 32 to 256 MB, filled from // pair A, so a plain total cannot describe it — refit // whatever the previous profile had. - self.cpu = CpuModel::R5000; + self.cpu = MachineProfile::O2Ip32.cpus()[0]; self.use_embedded_prom = false; self.prom_path = "ip32prom.bin".into(); self.ram_total_mb = 128; self.ram_banks = iris::config::banks_to4(crate::ram::o2_kits_for(128)); self.resolution = NewportResolution::Guest; } else if profile_before == MachineProfile::Indigo2Ip28 { - self.cpu = CpuModel::default(); + self.cpu = self.profile.cpus()[0]; self.ram_total_mb = self.ram_total_mb.min(512); // Banks above 128 MB require the IP28 MC. for bank in &mut self.ram_banks { @@ -156,7 +157,7 @@ impl NewMachineDialog { } else if profile_before == MachineProfile::O2Ip32 { // Leaving the O2: a total in the MC's granule, and // banks the Indy/Indigo2 can actually express. - self.cpu = CpuModel::default(); + self.cpu = self.profile.cpus()[0]; self.ram_total_mb = self.ram_total_mb.min(512); self.ram_banks = iris::config::banks_to4( distribute_ram(self.ram_total_mb, self.profile)); @@ -167,9 +168,8 @@ impl NewMachineDialog { let octane = self.profile == MachineProfile::OctaneIp30; let o2 = self.profile == MachineProfile::O2Ip32; ui.label(t!("dialog-new-machine-cpu")); - // Only the CPUs this machine can be built with. The Octane - // has one (the 195 MHz R10000), so its row is not a choice - // at all; an O2 refuses an R4400. + // Only the CPUs this machine can be built with; a machine + // with one has no choice. let cpus = self.profile.cpus(); ui.add_enabled_ui(cpus.len() > 1, |ui| { ComboBox::from_id_salt("nm_cpu") @@ -180,7 +180,7 @@ impl NewMachineDialog { } }); }); - if octane { + if cpus.len() == 1 { ui.label(RichText::new(cpus[0].label()).weak()); } ui.end_row(); @@ -381,7 +381,7 @@ impl NewMachineDialog { { let mut cfg = MachineConfig::default(); cfg.machine.profile = self.profile; - cfg.machine.cpu = self.cpu; + cfg.set_cpu(self.cpu); cfg.graphics.resolution = self.resolution; cfg.prom = if self.use_embedded_prom { String::new() } else { self.prom_path.clone() }; // Empty prom path makes Machine::new fall back to embedded @@ -451,7 +451,7 @@ mod ram_tests { for &total in crate::ram::ram_presets(octane) { let mut cfg = iris::config::MachineConfig::default(); cfg.machine.profile = octane; - cfg.machine.cpu = iris::config::CpuModel::R10000; + cfg.set_cpu(octane.cpus()[0]); cfg.banks = distribute_ram(total, octane); cfg.validate().unwrap_or_else(|e| panic!("{total} MB: {e}")); } diff --git a/iris-gui/src/main.rs b/iris-gui/src/main.rs index 2b4e78fc5..f5f1402f0 100644 --- a/iris-gui/src/main.rs +++ b/iris-gui/src/main.rs @@ -244,7 +244,7 @@ struct App { /// Banks passed to the last `Cmd::Start` (guest-visible RAM is fixed until Stop). started_banks: Option<[u32; iris::config::MAX_BANKS]>, /// The CPU the running guest booted on — see `MemoryUiContext::started_cpu`. - started_cpu: Option, + started_cpu: Option<&'static iris::cpu::parts::Part>, tab: Tab, /// Benchmark tab state: the child process, its streamed output, and the /// last outcome. Lives here rather than in the tab function because a run @@ -829,7 +829,7 @@ impl App { (iris::config::NetMode::Nat, None) }); self.started_banks = Some(self.cfg.banks); - self.started_cpu = Some(self.cfg.machine.cpu); + self.started_cpu = Some(self.cfg.cpu()); // Don't resize the window when the VM launches — its size is latched at // app load (the saved window size, or the first-launch fit to vm_scale) // and the guest display is letterboxed into it. Only the VM-scale slider @@ -1339,12 +1339,11 @@ impl App { // Chosen at Machine::new, so an edit while running is pending, not live — // same shape as the Memory menu's RAM rows. ui.set_min_width(240.0); - ui.label(RichText::new(t!("menu-machine-processor", cpu = self.cfg.machine.cpu.label())).strong()); - // The Octane is an R10000 and nothing else. - ui.add_enabled_ui(!running && self.cfg.machine.profile != iris::config::MachineProfile::OctaneIp30, |ui| { - for c in iris::config::CpuModel::ALL { - if ui.radio(self.cfg.machine.cpu == c, c.label()).clicked() { - self.cfg.machine.cpu = c; + ui.label(RichText::new(t!("menu-machine-processor", cpu = self.cfg.cpu().label())).strong()); + ui.add_enabled_ui(!running && self.cfg.machine.profile.cpus().len() > 1, |ui| { + for &c in self.cfg.machine.profile.cpus() { + if ui.radio(self.cfg.cpu() == c, c.label()).clicked() { + self.cfg.set_cpu(c); self.mark_dirty(); self.toast(t!("toast-cpu-set", cpu = c.label())); ui.close(); @@ -1353,7 +1352,7 @@ impl App { }); if running { match self.started_cpu { - Some(started) if started != self.cfg.machine.cpu => { + Some(started) if started != self.cfg.cpu() => { ui.label(RichText::new(t!("menu-machine-running-pending", cpu = started.label())) .color(Color32::YELLOW)); } @@ -3073,10 +3072,10 @@ impl App { // Guest-visible (PRId, cache geometry, ISA level), so it belongs in // the summary next to the platform rather than buried in a tab. ui.label(t!("welcome-processor")); - ui.label(self.cfg.machine.cpu.label()); + ui.label(self.cfg.cpu().label()); ui.end_row(); if let Some(started) = self.started_cpu { - if started != self.cfg.machine.cpu { + if started != self.cfg.cpu() { ui.label(t!("welcome-processor-running")); ui.label(started.label()); ui.end_row(); @@ -3171,16 +3170,16 @@ impl App { // While running, name the CPU the guest actually booted on — the config // may already have been changed to one that only takes effect next Start. let running_cpu = if self.emu.is_running() { self.started_cpu } else { None }; - let cpu = running_cpu.unwrap_or(self.cfg.machine.cpu); + let cpu = running_cpu.unwrap_or(self.cfg.cpu()); ui.label(if self.cfg_dirty { t!("status-machine-unsaved", name = &name, profile = profile, cpu = cpu.label()) } else { t!("status-machine", name = &name, profile = profile, cpu = cpu.label()) }); if let Some(started) = running_cpu { - if started != self.cfg.machine.cpu { + if started != self.cfg.cpu() { ui.label( - RichText::new(t!("status-cpu-pending", cpu = self.cfg.machine.cpu.label())) + RichText::new(t!("status-cpu-pending", cpu = self.cfg.cpu().label())) .color(Color32::YELLOW).small(), ); } diff --git a/src/bench_runner.rs b/src/bench_runner.rs index 3461b861f..c97cefb7d 100644 --- a/src/bench_runner.rs +++ b/src/bench_runner.rs @@ -22,7 +22,8 @@ use parking_lot::Mutex; use crate::bench_report::{host_info, parse_block, Run}; use crate::benchsuite; -use crate::config::{CpuModel, JitCompile, MachineConfig}; +use crate::config::{JitCompile, MachineConfig}; +use crate::cpu::parts::{self, Part}; use crate::machine::Machine; use crate::dev::testdev::{RunConfig, TestDevice}; @@ -59,7 +60,7 @@ pub struct BenchOptions { pub timeout: Duration, /// Emulated CPU. Runtime config since 96e5ddd, so the in-process runner picks /// it like any other machine setting rather than needing its own build. - pub cpu: CpuModel, + pub cpu: &'static Part, /// Cache model, as `[machine] cache`. pub cache: crate::config::CacheModel, /// Make two runs do the same work: compile on the CPU thread @@ -81,7 +82,7 @@ impl Default for BenchOptions { quick: false, label: "local".to_string(), banks: BENCH_BANKS, - cpu: CpuModel::default(), + cpu: &parts::R4400_150, cache: crate::config::CacheModel::default(), deterministic: false, timeout: Duration::from_secs(1800), @@ -310,10 +311,20 @@ impl Table { } } +/// The parts the suite measures. It runs on an Indy with the test device on, +/// which takes any part. +pub const BENCH_CPUS: &[&Part] = &[ + &parts::R4400_150, + &parts::R5000_180_EARLY, + &parts::R10000_195, + &parts::R12000_360, + &parts::R14000_600, +]; + /// The bare-metal machine the suite runs on: RAM, a test device, and nothing /// else. No SCSI (there is no disk image and no filesystem to find one on), no /// graphics, no audio. -pub fn bench_config(banks: [u32; 4], cpu: CpuModel) -> MachineConfig { +pub fn bench_config(banks: [u32; 4], cpu: &Part) -> MachineConfig { let mut cfg = MachineConfig { banks: crate::config::banks_from4(banks), headless: true, @@ -324,7 +335,7 @@ pub fn bench_config(banks: [u32; 4], cpu: CpuModel) -> MachineConfig { // when the file is absent — and here it always is. Same reason // bench/run/bare.toml carries a present-but-empty `[scsi]`. cfg.scsi.clear(); - cfg.machine.cpu = cpu; + cfg.set_cpu(cpu); cfg } @@ -416,7 +427,7 @@ mod tests { &BenchOptions { quick: true, label: "test".into(), - cpu: CpuModel::R10000, + cpu: &parts::R10000_195, ..Default::default() }, |_| {}, @@ -496,7 +507,7 @@ mod tests { let on_exit = { let done = done.clone(); Box::new(move |_| done.store(true, Ordering::Release)) }; let testdev = Arc::new( TestDevice::new_embedded(sink.clone(), on_exit, RunConfig::ALL)); - let mut cfg = bench_config([32, 0, 0, 0], CpuModel::default()); + let mut cfg = bench_config([32, 0, 0, 0], &parts::R4400_150); if deterministic { cfg.jitv2.compile = JitCompile::Sync; } @@ -536,7 +547,7 @@ mod tests { #[test] fn the_bench_machine_has_no_disks() { - let cfg = bench_config(BENCH_BANKS, CpuModel::default()); + let cfg = bench_config(BENCH_BANKS, &parts::R4400_150); assert!(cfg.scsi.is_empty(), "a bench machine must not try to open a disk image"); assert!(cfg.headless && cfg.no_audio); } diff --git a/src/bin/iris_bench.rs b/src/bin/iris_bench.rs index 82ca50ba6..16e5f9ee8 100644 --- a/src/bin/iris_bench.rs +++ b/src/bin/iris_bench.rs @@ -32,6 +32,7 @@ use iris::bench_report::{ RunSettings, EXPECT_EXC, }; use iris::bench_runner::{self, BenchOptions}; +use iris::cpu::parts::{self, Part}; // ─── running ───────────────────────────────────────────────────────────────── @@ -140,6 +141,11 @@ fn run_host(exe: &Path, timeout_s: u64) -> Result { }) } +/// `--cpu`: a part or a family of the ones the suite measures. +fn bench_cpu(name: &str) -> Result<&'static Part, String> { + parts::find(bench_runner::BENCH_CPUS, name).ok_or_else(|| format!("unknown cpu {name:?}")) +} + /// Run the suite inside this process, streaming the guest's console as it /// arrives. /// @@ -147,7 +153,7 @@ fn run_host(exe: &Path, timeout_s: u64) -> Result { /// for a minute is not mistaken for a hang, and that property is worth keeping /// at the command line — so this echoes every line rather than waiting for the /// end and printing the parsed summary. -fn run_embedded(label: &str, timeout_s: u64, quick: bool, cpu: iris::config::CpuModel, +fn run_embedded(label: &str, timeout_s: u64, quick: bool, cpu: &'static Part, cache: iris::config::CacheModel, deterministic: bool) -> Result { let opts = BenchOptions { quick, @@ -271,6 +277,12 @@ const CELLS: &[Cell] = &[ Cell { name: "r5000-jitv2", features: "jitv2", cpu: "r5000", expect_cpu: "R5000" }, Cell { name: "r4400-lightning", features: "lightning", cpu: "r4400", expect_cpu: "R4400" }, Cell { name: "r4400-jitv2-lightning", features: "jitv2,lightning", cpu: "r4400", expect_cpu: "R4400" }, + // The R10000 family. It always runs the shadow caches, so the default + // build covers it. Full chip names: a bare family name would pick the + // chip of the first machine that lists it. + Cell { name: "r10000-interp", features: "", cpu: "r10000_195", expect_cpu: "R10000" }, + Cell { name: "r12000-interp", features: "", cpu: "r12000_360", expect_cpu: "R12000" }, + Cell { name: "r14000-interp", features: "", cpu: "r14000_600", expect_cpu: "R14000" }, ]; fn build_cell(cell: &Cell, root: &Path, force: bool) -> Result { @@ -724,8 +736,8 @@ enum Cmd { quick: bool, /// Emulated CPU for the in-process run. With --iris, forward it to the /// emulator instead: `-- --cpu r5000`. - #[arg(long, default_value = "r4400")] - cpu: iris::config::CpuModel, + #[arg(long, default_value = "r4400", value_parser = bench_cpu)] + cpu: &'static Part, /// Cache model for the in-process run: auto (shadow in a jitv2 /// build), full or shadow. With --iris: `-- --cache shadow`. #[arg(long, default_value = "auto")] @@ -1092,13 +1104,13 @@ mod tests { #[test] fn every_cell_name_is_unique_and_maps_to_a_cpu() { for (i, a) in CELLS.iter().enumerate() { - assert!(a.expect_cpu == "R4400" || a.expect_cpu == "R5000"); // `expect_cpu` is what the `matrix` guard compares the guest's // `#machine cpu=` line against, so it must describe the `--cpu` // this cell actually passes. CPU is a runtime flag (see `Cell::cpu`), // not a cargo feature, so the two agreeing is the whole contract. - let want = match a.cpu { "r4400" => "R4400", "r5000" => "R5000", c => panic!("{}: unknown --cpu {}", a.name, c) }; + let want = a.cpu.split('_').next().unwrap().to_uppercase(); assert_eq!(a.expect_cpu, want, "{}", a.name); + assert!(bench_cpu(a.cpu).is_ok()); for b in CELLS.iter().skip(i + 1) { assert_ne!(a.name, b.name); } } } diff --git a/src/config.rs b/src/config.rs index 39ca0ab42..6ec94facd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,8 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use serde::{Deserialize, Serialize}; use std::net::Ipv4Addr; +use crate::cpu::parts::{self, Part}; + /// Every memory bank size in MB, for the front-ends' menus; `bank_sizes` says /// which of them a machine takes. pub const VALID_BANK_SIZES: &[u32] = &[0, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]; @@ -706,15 +708,17 @@ impl MachineProfile { } } - /// Whether this profile can have `cpu`, and why not if it cannot. The - /// Octane builds only the part below; the O2 constructor still refuses - /// its own, after `validate()`. - pub fn accepts_cpu(self, cpu: CpuModel) -> Result<(), String> { - match self { - Self::OctaneIp30 if cpu != CpuModel::R10000 => { - Err("an Octane has an R10000 ([machine] cpu = \"r10000\")".into()) - } - _ => Ok(()), + /// The processor `cpu` names on this machine (none: the default, the + /// first of `cpus()`), or the error that names the ones it takes. A family + /// name means the machine's part of that family. + pub fn part(self, cpu: Option<&str>) -> Result<&'static Part, String> { + let cpus = self.cpus(); + match cpu { + None => Ok(cpus[0]), + Some(name) => parts::find(cpus, name).ok_or_else(|| { + let names: Vec<_> = cpus.iter().map(|p| p.name()).collect(); + format!("{} takes [machine] cpu = {}", self.label(), names.join(" or ")) + }), } } @@ -761,18 +765,14 @@ impl MachineProfile { matches!(self, Self::IndyIp24) } - /// The CPU models this machine can be built with. The O2 has an R5000 or - /// an R10000 (an R4400 is refused when the machine is built, not by - /// `validate()`); the Octane shipped only with the 195 MHz R10000; the - /// Indy and the Indigo2 take any of the three. - /// - /// Single source of truth for the front-ends' Processor pickers, so they - /// cannot offer a CPU the machine will refuse. - pub fn cpus(self) -> &'static [CpuModel] { + /// The processors this machine shipped with, the default first. Single + /// source of truth for `validate` and the front-ends' Processor pickers. + pub fn cpus(self) -> &'static [&'static Part] { match self { - Self::OctaneIp30 => &[CpuModel::R10000], - Self::O2Ip32 => &[CpuModel::R5000, CpuModel::R10000], - _ => &CpuModel::ALL, + Self::OctaneIp30 => crate::ip30_machine::CPUS, + Self::O2Ip32 => crate::ip32_machine::CPUS, + Self::Indigo2Ip28 => crate::machine::IP28_CPUS, + Self::IndyIp24 | Self::Indigo2Ip22 => crate::machine::CPUS, } } @@ -1065,35 +1065,6 @@ impl KeyboardLayout { } } -/// Emulated CPU. Runtime-selectable: each model is its own monomorphisation, -/// so the hot path carries no per-model branch. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, clap::ValueEnum)] -#[serde(rename_all = "snake_case")] -#[clap(rename_all = "lower")] -pub enum CpuModel { - /// MIPS R4400, 16K direct-mapped L1s, 1 MB L2. The Indy IRIS has always shipped. - #[default] - R4400, - /// MIPS R5000, 2-way 32K L1s, no secondary cache, MIPS IV. - R5000, - /// MIPS R10000, 2-way 32K L1s, 1 MB secondary cache, MIPS IV. The CPU in - /// the Indigo2 IMPACT (IP28). The machine uses shadow cache arrays; - /// see `docs/indigo2-ip28.md` and `src/cpu/mips_cache_shadow.rs`. - R10000, -} - -impl CpuModel { - pub const ALL: [Self; 3] = [Self::R4400, Self::R5000, Self::R10000]; - - pub fn label(self) -> &'static str { - match self { - Self::R4400 => "MIPS R4400", - Self::R5000 => "MIPS R5000", - Self::R10000 => "MIPS R10000", - } - } -} - /// How the CPU's caches are modelled. A runtime choice: each is its own /// monomorphisation, picked when the machine is built. /// @@ -1105,7 +1076,7 @@ impl CpuModel { /// need no tag check against it. /// - `auto`: `shadow` in a `jitv2` build, `full` otherwise. /// -/// The R10000 (IP28) only has the shadow model. +/// The R10000 family only has the shadow model. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, clap::ValueEnum)] #[serde(rename_all = "snake_case")] #[clap(rename_all = "lower")] @@ -1118,9 +1089,9 @@ pub enum CacheModel { impl CacheModel { /// Whether `cpu` runs the shadow cache under this setting. - pub fn is_shadow(self, cpu: CpuModel) -> bool { + pub fn is_shadow(self, cpu: &Part) -> bool { match (cpu, self) { - (CpuModel::R10000, _) => true, + (cpu, _) if cpu.is_r10000_family() => true, (_, Self::Full) => false, (_, Self::Shadow) => true, (_, Self::Auto) => cfg!(feature = "jitv2"), @@ -1134,8 +1105,10 @@ impl CacheModel { pub struct MachineSection { #[serde(default)] pub profile: MachineProfile, - #[serde(default)] - pub cpu: CpuModel, + /// A part (`r14000_600`) or a family (`r14000`); see `MachineConfig::cpu_part`. + /// None is the machine's first. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu: Option, #[serde(default)] pub cache: CacheModel, /// Processors fitted. More than one only on the Octane. @@ -1149,7 +1122,7 @@ fn default_cpus() -> u8 { impl Default for MachineSection { fn default() -> Self { - Self { profile: MachineProfile::default(), cpu: CpuModel::default(), cache: CacheModel::default(), cpus: default_cpus() } + Self { profile: MachineProfile::default(), cpu: None, cache: CacheModel::default(), cpus: default_cpus() } } } @@ -1425,15 +1398,21 @@ impl Jitv2Config { #[serde(deny_unknown_fields)] pub struct ClockConfig { /// CP0 Count frequency in MHz. Count ticks at a fixed rate; there is no - /// runtime inference. None selects the profile default: 33 MHz on - /// IP22/IP24, 97.5 MHz on IP28, 90 MHz on the O2 (a 180 MHz CPU). IRIX - /// reports twice the Count rate as CPU MHz, and the O2's PROM measures - /// it the same way; host emulation throughput is independent of this - /// setting. + /// runtime inference. None selects half the configured part's clock + /// (`Part::count_hz`). IRIX reports twice the Count rate as CPU MHz, + /// and the O2's PROM measures it the same way; host emulation throughput + /// is independent of this setting. #[serde(default, skip_serializing_if = "Option::is_none")] pub fixed_mhz: Option, } +impl ClockConfig { + /// The Count rate for `cpu`: `fixed_mhz`, else half the part's clock. + pub fn count_hz(&self, cpu: &Part) -> u64 { + self.fixed_mhz.map_or(cpu.count_hz(), |mhz| (mhz * 1_000_000.0) as u64) + } +} + impl Default for ClockConfig { fn default() -> Self { Self { fixed_mhz: None } @@ -2368,9 +2347,31 @@ impl MachineConfig { self.graphics.board.unwrap_or_else(|| GraphicsBoard::default_for(self.machine.profile)) } + /// The processor `[machine] cpu` names, resolved against the final + /// machine profile. The bare-metal test rig (iris-bench) measures every + /// part on one machine, so with the test device on any part will do. + pub fn cpu_part(&self) -> Result<&'static Part, String> { + let named = self.machine.cpu.as_deref(); + match self.machine.profile.part(named) { + Err(_) if self.test_device => named.and_then(|n| parts::find(parts::ALL, n)) + .ok_or_else(|| format!("unknown cpu {:?}", named.unwrap_or_default())), + part => part, + } + } + + /// `cpu_part`, or the machine's default where the config names a part it + /// does not take, for front-ends that show a config `validate` will refuse. + pub fn cpu(&self) -> &'static Part { + self.cpu_part().unwrap_or(self.machine.profile.cpus()[0]) + } + + pub fn set_cpu(&mut self, part: &Part) { + self.machine.cpu = Some(part.name()); + } + pub fn validate(&self) -> Result<(), String> { self.validate_virtio_serial()?; - self.machine.profile.accepts_cpu(self.machine.cpu)?; + self.cpu_part()?; self.machine.profile.accepts_cpus(self.machine.cpus)?; if self.graphics.heads != 1 && self.graphics.heads != 2 { return Err(format!( @@ -2819,16 +2820,19 @@ pub struct Cli { #[arg(long = "no-scsi-deferred-int", default_value_t = false)] pub no_scsi_deferred_int: bool, - /// Emulated CPU: `r4400` (default), `r5000`, or `r10000`. Overrides `[machine] cpu`. + /// Emulated CPU: a family (`r4400` (default), `r5000`, `r10000`, `r12000` or `r14000`) or a chip (`r14000_600`). Overrides `[machine] cpu`. /// - /// All three models are compiled into every build — each is its own + /// Every model is compiled into every build — each is its own /// monomorphisation, so the hot path carries no per-model branch — and this /// picks between them at construction. The `r5k` Cargo feature is removed. + /// A family means the machine's chip of that family. The Indy and + /// `indigo2_ip22` take `r4400` and `r5000`; the IP28 `r10000`; the Octane + /// `r10000`, `r12000` and `r14000`; the O2 `r5000`, `r10000` and `r12000`. #[arg(long = "cpu", value_name = "MODEL")] - pub cpu: Option, + pub cpu: Option, /// Cache model: `auto` (default: `shadow` in a jitv2 build, else `full`), - /// `full` or `shadow`. Overrides `[machine] cache`. The R10000 is always + /// `full` or `shadow`. Overrides `[machine] cache`. The R10000 family is always /// `shadow`. #[arg(long = "cache", value_name = "MODEL")] pub cache: Option, @@ -3018,7 +3022,8 @@ impl Cli { } } - if let Some(cpu) = self.cpu { cfg.machine.cpu = cpu; } + // Resolved by `MachineConfig::cpu_part` once the profile is final. + if let Some(cpu) = &self.cpu { cfg.machine.cpu = Some(cpu.clone()); } if let Some(cache) = self.cache { cfg.machine.cache = cache; } if let Some(b) = self.graphics { cfg.graphics.board = Some(b); } if let Some(r) = self.raster { cfg.graphics.raster = r; } @@ -3400,7 +3405,7 @@ mod export_tests { cfg.jitv2.compile = JitCompile::Sync; cfg.validate().expect("sync compile on the default machine"); cfg.machine.profile = MachineProfile::OctaneIp30; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.machine.cpus = 2; assert!(cfg.validate().unwrap_err().contains("one CPU")); } @@ -3430,7 +3435,7 @@ mod export_tests { assert!(cfg.validate().unwrap_err().contains("512 MB needs the IP28")); } cfg.machine.profile = MachineProfile::Indigo2Ip28; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.validate().expect("IP28 accepts two 512 MB banks"); } @@ -3443,7 +3448,7 @@ mod export_tests { assert!(err.contains("256 MB"), "{err}"); { cfg.machine.profile = MachineProfile::Indigo2Ip28; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.validate().expect("the IP28 MC can"); } } @@ -3455,14 +3460,14 @@ mod export_tests { fn o2_takes_scsi_tables_with_controller_as_bus() { let mut cfg = MachineConfig::default(); cfg.machine.profile = MachineProfile::O2Ip32; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.scsi.insert(2, ScsiDeviceConfig { path: "external.raw".into(), controller: 1, ..Default::default() }); cfg.validate().expect("the O2 has two SCSI controllers"); cfg.machine.profile = MachineProfile::IndyIp24; - cfg.machine.cpu = CpuModel::default(); + cfg.machine.cpu = None; assert!(cfg.validate().unwrap_err().contains("controller 1"), "the Indy has one"); cfg.machine.profile = MachineProfile::O2Ip32; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.scsi.insert(3, ScsiDeviceConfig { kind_field: ScsiKind::Daynaport, ..Default::default() }); assert!(cfg.validate().unwrap_err().contains("daynaport")); } @@ -3473,7 +3478,7 @@ mod export_tests { for profile in [MachineProfile::default(), MachineProfile::O2Ip32, MachineProfile::OctaneIp30] { let mut cfg = MachineConfig::default(); cfg.machine.profile = profile; - if profile != MachineProfile::default() { cfg.machine.cpu = CpuModel::R10000; } + if profile != MachineProfile::default() { cfg.set_cpu(profile.cpus()[0]); } cfg.network.mode = NetMode::Stream; assert!(cfg.validate().unwrap_err().contains("needs [network] stream"), "{profile:?}"); cfg.network.stream = Some("tcp:nope".into()); @@ -3506,7 +3511,7 @@ mod export_tests { let mut o2 = MachineConfig::default(); o2.machine.profile = MachineProfile::O2Ip32; - o2.machine.cpu = CpuModel::R10000; + o2.machine.cpu = Some("r10000".into()); o2.validate().expect("two 128 MB kits are an O2's 256 MB"); o2.banks = banks_from4([256, 256, 256, 256]); o2.validate().expect("1 GB"); @@ -3594,9 +3599,9 @@ mod export_tests { /// and the Octane constructor refuse a bad processor with the same words. #[test] fn a_profile_says_which_processors_it_accepts() { - let accepted = |p: MachineProfile| CpuModel::ALL.into_iter().filter(|&c| p.accepts_cpu(c).is_ok()).collect::>(); - assert_eq!(accepted(MachineProfile::OctaneIp30), [CpuModel::R10000]); - assert_eq!(accepted(MachineProfile::IndyIp24), [CpuModel::R4400, CpuModel::R5000, CpuModel::R10000]); + let accepts = |p: MachineProfile, cpu: &str| p.part(Some(cpu)).is_ok(); + assert!(accepts(MachineProfile::OctaneIp30, "r14000") && !accepts(MachineProfile::OctaneIp30, "r5000")); + assert!(accepts(MachineProfile::IndyIp24, "r5000") && !accepts(MachineProfile::IndyIp24, "r10000")); // The counts are what the checks in validate() and the Octane // constructor allowed before they moved here: one on every machine @@ -3616,20 +3621,20 @@ mod export_tests { let mut octane = MachineConfig::default(); octane.machine.profile = MachineProfile::OctaneIp30; - octane.machine.cpu = CpuModel::R10000; + octane.machine.cpu = Some("r10000".into()); octane.banks = banks_from4([128, 0, 0, 0]); octane.machine.cpus = 2; octane.validate().expect("two processors"); octane.machine.cpus = 3; assert!(octane.validate().unwrap_err().contains("1 or 2")); octane.machine.cpus = 1; - octane.machine.cpu = CpuModel::R4400; - assert!(octane.validate().unwrap_err().contains("an Octane has an R10000")); + octane.machine.cpu = Some("r4400".into()); + assert!(octane.validate().unwrap_err().contains("takes [machine] cpu = r10000_195")); // The constructor refuses a processor with the words validate() does. let mut cfg = MachineConfig::default(); cfg.machine.profile = MachineProfile::OctaneIp30; - cfg.machine.cpu = CpuModel::R4400; + cfg.machine.cpu = Some("r4400".into()); let from_validate = cfg.validate().unwrap_err(); assert_eq!(crate::ip30_machine::OctaneMachine::new(&cfg).err(), Some(from_validate)); } diff --git a/src/cpu/build.rs b/src/cpu/build.rs new file mode 100644 index 000000000..7a8e1f0e9 --- /dev/null +++ b/src/cpu/build.rs @@ -0,0 +1,28 @@ +//! Building the configured processor: the one place a [`Part`] becomes a +//! cache and executor type. +//! +//! [`Part`]: crate::cpu::parts::Part + +/// Run `$body` with `$C` the cache type of `$part`, chosen once at machine +/// construction so the hot path carries no per-model branch. `$shadow` picks +/// the shadow caches for the R4400 and R5000, which also have the functional +/// ones (`CacheModel`). `$o2` selects the O2's R10000 and R12000 types, whose +/// secondary caches have 64-byte lines where the others have 128. +#[macro_export] +macro_rules! with_cpu { + ($part:expr, $shadow:expr, $o2:expr, $C:ident => $body:expr) => {{ + use $crate::cpu::mips_cache_shadow as sh; + use $crate::cpu::mips_cache_v2 as full; + use $crate::cpu::parts::Family as F; + match ($part.family, $shadow, $o2) { + (F::R4400, true, _) => { type $C = sh::R4400ShadowCache; $body } + (F::R4400, false, _) => { type $C = full::R4400Cache; $body } + (F::R5000, true, _) => { type $C = sh::R5000ShadowCache; $body } + (F::R5000, false, _) => { type $C = full::R5000Cache; $body } + (F::R10000, _, true) => { type $C = sh::R10000O2ShadowCache; $body } + (F::R10000, _, false) => { type $C = sh::R10000ShadowCache; $body } + (F::R12000 | F::R14000, _, true) => { type $C = sh::R12000O2ShadowCache; $body } + (F::R12000 | F::R14000, _, false) => { type $C = sh::R12000ShadowCache; $body } + } + }}; +} diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index d79e8d410..c50605acc 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -36,7 +36,7 @@ use std::sync::Arc; use crate::cpu::mips_cache_v2::{ cache_op_name, model, r4k_load_taglo, r4k_store_taglo, CpuModel, FetchInstrResult, MipsCache, - C_ILT, C_IST, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, + C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, }; use crate::traits::{BusDevice, BusRead64, BUS_OK}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_OP, devlog_is_active, devlog_mask}; @@ -183,8 +183,12 @@ impl< const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, > ShadowCache { - const R10K: bool = MODEL == model::R10000; + const R10K: bool = model::r10k_family(MODEL); const R5K: bool = MODEL == model::R5000; + /// Index Writeback Invalidate (D) also zeroes the primary data tag on the + /// R12000 and later; the R10000 leaves it as it was. NEC VR12000 user's + /// manual, Appendix B.20. The R14000 is taken to match. + const IWBI_CLEARS_PD_TAG: bool = Self::R10K && MODEL != model::R10000; const IC_LINES: usize = if IC_LINE == 0 { 0 } else { IC_SIZE / IC_LINE }; const DC_LINES: usize = if DC_LINE == 0 { 0 } else { DC_SIZE / DC_LINE }; const L2_LINES: usize = if L2_LINE == 0 { 0 } else { L2_SIZE / L2_LINE }; @@ -362,11 +366,10 @@ impl< const PRID: u32 = PRID; const FIR: u32 = FIR; const TLB_ENTRIES: usize = TLB_ENTRIES; - // The R10000 implements 44 virtual address bits where the R4x00 implements - // 40; the shadow cache is only used for it, but key this off the same flag - // that selects its cache encodings rather than asserting it unconditionally. - const VA_BITS: u32 = if MODEL == model::R10000 { 44 } else { 40 }; - const PA_BITS: u32 = if MODEL == model::R10000 { 40 } else { 36 }; + // The R10000 family implements 44 virtual address bits where the R4x00 + // implements 40. + const VA_BITS: u32 = if Self::R10K { 44 } else { 40 }; + const PA_BITS: u32 = if Self::R10K { 40 } else { 36 }; // The CPU's name, which snapshots check: the same as the full model's, so // a mismatch reads as a cache-kind difference rather than a different CPU. // The R10000 keeps its historical "shadow", which existing IP28 @@ -374,10 +377,11 @@ impl< const NAME: &'static str = match MODEL { model::R4400 => "R4400", model::R5000 => "R5000", + model::R12000 => "R12000", _ => "shadow", }; const CACHE_KIND: &'static str = "shadow"; - const R10K_CACHE_OPS: bool = MODEL == model::R10000; + const R10K_CACHE_OPS: bool = Self::R10K; } impl< @@ -658,6 +662,15 @@ impl< 0 } + C_IWBINV if Self::IWBI_CLEARS_PD_TAG && sel == CACH_PD => { + let idx = self.tag_slot(sel, virt_addr); + let s = self.shadow(sel); + if idx < s.tags.len() { + s.tags[idx] = 0; + } + 0 + } + // Invalidate, writeback, fill, and the R4000 hit operations. All // no-ops: there is nothing held that could be stale or dirty. _ => 0, @@ -758,6 +771,24 @@ impl< pub type R10000ShadowCache = ShadowCache<32768, 64, 32768, 32, 1048576, 128, true, 0x0000_0925, 0x0000_0900, 64, { model::R10000 }>; +/// R12000 3.5 as the Octane has it: the R10000's programming model with a +/// 2 MB secondary cache. PRId 0x0e35, FIR 0x900, 128-byte secondary lines +/// (SB: a model choice). +/// +/// The Octane's R14000 is this type with its own PRId set at build time (see +/// `Part::prid`), as the Octane's own R10000 revision is; only the PRId +/// differs. Both report "R12000" as `NAME`. +pub type R12000ShadowCache = + ShadowCache<32768, 64, 32768, 32, 2097152, 128, true, 0x0000_0e35, 0x0000_0900, 64, { model::R12000 }>; + +/// R12000 3.5 as the O2 has it: the Octane part with 64-byte secondary lines +/// (SB clear). With 128-byte lines the O2 PROM panics before its banner +/// (write TLB miss at 0x0); 64 boots. IRIX on the O2 never reads SB and sizes +/// the secondary cache as 512 KB << SS. +pub type R12000O2ShadowCache = + ShadowCache<32768, 64, 32768, 32, 2097152, 64, true, 0x0000_0e35, 0x0000_0900, 64, { model::R12000 }>; + + /// SGI O2 R10000 (IP32): the IP28 model with 64-byte secondary-cache lines. /// /// The O2 PROM (4.18 and earlier) handles only that line size. With Config's @@ -977,6 +1008,21 @@ mod tests { assert_eq!(c.read::<4>(0, 0x2000).data, 0x1234_5678); } + /// On the R12000 and later Index Writeback Invalidate (D) zeroes the line's + /// primary data tag; the R10000 keeps it. + #[test] + fn index_writeback_invalidate_d_clears_the_tag_from_the_r12000_on() { + fn run>>() -> u64 { + let mem: Arc = Arc::new(Memory::new(1)); + let c = C::from(mem); + c.cache_op(crate::cpu::mips_cache_v2::C_IST | CACH_PD, 0x40, 0x1_2345_6780); + c.cache_op(crate::cpu::mips_cache_v2::C_IWBINV | CACH_PD, 0x40, 0); + c.cache_op(crate::cpu::mips_cache_v2::C_ILT | CACH_PD, 0x40, 0) + } + assert_eq!(run::(), 0); + assert_eq!(run::(), 0x1_2345_6780); + } + /// Geometry is what CP0 Config is built from, so it must be the real /// part's even though the model holds nothing. #[test] diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 448c47726..2b92dc849 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -642,8 +642,8 @@ pub trait CpuModel: MipsCache { /// The part is in the R10000 family, which shares one programming model: /// FrameMask, and 44 virtual and 40 physical address bits. /// Established for the R10000. The R12000 is inferred to match: its manual's - /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and - /// R16000 are a model choice that gives them the same TLB. + /// list of changes from the R10000 has no TLB or CP0 change. The R14000 is + /// a model choice that gives it the same TLB. const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; /// A partial LWR keeps bits 63:32 of the destination. Observed on the /// R10000; whether the later parts do the same is not established, so they @@ -1483,6 +1483,13 @@ pub mod model { pub const R4400: u8 = 0; pub const R5000: u8 = 1; pub const R10000: u8 = 2; + pub const R12000: u8 = 3; + + /// The R10000 family: the R10000 and the parts that follow it, which share + /// one programming model. + pub const fn r10k_family(model: u8) -> bool { + matches!(model, R10000 | R12000) + } } /// SGI Indy R4400: direct-mapped 16K L1s, 1 MB unified L2 owning the decode slots. diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index c93b93b81..7a5edf710 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -707,13 +707,16 @@ pub struct MipsCpuConfig { pub tlb_entries: usize, /// Where the machine has memory. pub memory: Arc, + /// The PRId a guest reads: the cache type's unless the machine sets its + /// chip's (`with_prid`). + pub prid: Option, } impl MipsCpuConfig { /// Default configuration matching SGI Indy (R4400): 48-entry TLB. /// Cache geometry is fixed at compile time via constants in mips_cache_v2. pub fn indy() -> Self { - Self { tlb_entries: 48, memory: Arc::new(crate::ip22_bus::memory_map(false)) } + Self { tlb_entries: 48, memory: Arc::new(crate::ip22_bus::memory_map(false)), prid: None } } /// The JTLB size the CPU model declares. @@ -725,7 +728,12 @@ impl MipsCpuConfig { /// 48, and SGI's IP28 diagnostic writes index 48 on its first cache-alias /// test — every one of its reported failures was that write going nowhere. pub fn for_model(memory: Arc) -> Self { - Self { tlb_entries: C::TLB_ENTRIES, memory } + Self { tlb_entries: C::TLB_ENTRIES, memory, prid: None } + } + + pub fn with_prid(mut self, prid: u32) -> Self { + self.prid = Some(prid); + self } } @@ -2331,15 +2339,10 @@ impl MipsExecutor { c |= log2(32768) << 26; // 32 KB L1D c |= 1 << 15; // big endian if C::L2_LINE == 128 { c |= 1 << 13; } - // Secondary cache size. The encoding is not in anything to hand, - // so it was swept against the PROM rather than guessed. - // - // 1 is the value to keep: IRIX reports "Secondary unified - // instruction/data cache size: 1 Mbyte", which agrees with this - // model's own `L2_SIZE`, and the PROM's power-on diagnostics pass - // and IRIX boots with it. 4 also passes POST but has IRIX report - // 8 MB, contradicting the model. - c |= 1 << 16; + // SS is log2(secondary cache size / 512 KB): 0 is 512 KB, 1 is 1 MB, + // 2 is 2 MB, up to 5 for 16 MB (NEC U10278EJ4V0UM p.230). It follows + // the model's own `L2_SIZE`. EC, SC, DN, PM, DSD and SK stay zero. + c |= (C::L2_SIZE / (512 << 10)).trailing_zeros() << 16; c |= 2; // K0 = uncached at reset config = c; } @@ -2349,9 +2352,10 @@ impl MipsExecutor { // MipsCore::new already ran reset_registers, so set both the reset value // and the live register — otherwise the model's identity only appears // after the next power_on. - core.reset_prid = C::PRID; + let prid = cfg.prid.unwrap_or(C::PRID); + core.reset_prid = prid; core.reset_fir = C::FIR; - core.cp0_prid = C::PRID; + core.cp0_prid = prid; core.fpu_fir = C::FIR; // Triton: sync initial L2 enabled state from Config SE bit (starts 0 = disabled). @@ -4760,7 +4764,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", const EH_REGION: u64 = 0xC000_0000_0000_0000; // VPN2 is bits 43:13 on the R10000 family (R10000 UM pp. 219, 288) and // 39:13 on the R4x00 and R5000. The R12000 is inferred to match (its - // manual lists no TLB change); the R14000 and R16000 are a model choice + // manual lists no TLB change); the R14000 is a model choice // that follows the same width. let eh_vpn2_64 = crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS); diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index c23d015ef..e89ae4979 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -6683,4 +6683,21 @@ mod tests { check::(true); check::(false); } + + /// Config SS (bits 18:16) is log2(secondary size / 512 KB) and SB (bit 13) is + /// set for 128-byte lines (NEC U10278EJ4V0UM p.230): a 2 MB R12000 reads + /// SS=2, and the O2's 64-byte lines clear SB. Fix: Config reports each part. + #[test] + fn config_reports_the_secondary_cache_of_each_part() { + use crate::cpu::mips_cache_shadow::*; + fn ss_sb>>() -> (u64, u64) { + let c = tlb_exec::(Arc::new(MockMemory::new())).core.read_cp0(16); + ((c >> 16) & 7, (c >> 13) & 1) + } + on_big_stack(|| { + assert_eq!(ss_sb::(), (1, 1)); + assert_eq!(ss_sb::(), (2, 1)); + assert_eq!(ss_sb::(), (2, 0)); + }); + } } diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 166e2bc47..3eaed8e8c 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -361,7 +361,7 @@ pub trait Tlb { /// Make a TLB write invalidate every other entry it would match together /// with, as the R10000 and R12000 do (NEC U10278EJ4V0UM, Status register /// TS bit, p.223, and "Avoiding TLB Conflict", p.300): they do not shut - /// down on a multiple match. The R14000 and R16000 are given the same + /// down on a multiple match. The R14000 is given the same /// rule as a model choice, matching QEMU's r10k_invalidate_conflicts(); /// no manual for them was read. Off for the R4400 and R5000, whose /// behaviour on a multiple match is not modelled. diff --git a/src/cpu/mod.rs b/src/cpu/mod.rs index 276bc64bc..a70353f3b 100644 --- a/src/cpu/mod.rs +++ b/src/cpu/mod.rs @@ -8,6 +8,8 @@ pub mod mips_core; pub mod mips_tlb; pub mod mips_cache_v2; pub mod mips_cache_shadow; +pub mod build; +pub mod parts; pub mod decode_cache; pub mod mips_exec; pub mod cpu_monitor; diff --git a/src/cpu/parts.rs b/src/cpu/parts.rs new file mode 100644 index 000000000..598b923a8 --- /dev/null +++ b/src/cpu/parts.rs @@ -0,0 +1,104 @@ +//! The processors SGI shipped, one const each. A [`Part`] is a CPU as `hinv` +//! reports it, and everything iris knows about a part is here and nowhere +//! else: no machine keeps its own PRId, clock or revision. A machine lists the +//! parts it shipped with, the default first, as its own `CPUS`. +//! +//! The `_EARLY` suffix marks earlier silicon of the same family and clock; it +//! exists only where two shipped parts would otherwise share a name. + +/// A processor family, numbered as it names the parts (`r10000_195`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Family { + R4400 = 4400, + R5000 = 5000, + R10000 = 10000, + R12000 = 12000, + R14000 = 14000, +} + +impl Family { + /// PRId bits 15:8 (R10000 UM, PRId register). + const fn implementation(self) -> u32 { + match self { + Self::R4400 => 0x04, + Self::R5000 => 0x23, + Self::R10000 => 0x09, + Self::R12000 => 0x0e, + Self::R14000 => 0x0f, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Part { + pub family: Family, + pub mhz: u32, + /// Major and minor revision, as `hinv` prints them ("Revision: 2.7"). + pub rev: (u8, u8), + /// Secondary cache in KB. The R5000 reads 0: its cache is not modelled. + pub l2_kb: u32, +} + +pub const R4400_150: Part = Part { family: Family::R4400, mhz: 150, rev: (5, 0), l2_kb: 1024 }; +pub const R5000_180: Part = Part { family: Family::R5000, mhz: 180, rev: (2, 1), l2_kb: 0 }; +pub const R5000_180_EARLY: Part = Part { rev: (1, 0), ..R5000_180 }; +pub const R10000_195: Part = Part { family: Family::R10000, mhz: 195, rev: (2, 7), l2_kb: 1024 }; +pub const R10000_195_EARLY: Part = Part { rev: (2, 5), ..R10000_195 }; +pub const R10000_250: Part = Part { mhz: 250, rev: (3, 4), ..R10000_195 }; +pub const R12000_360: Part = Part { family: Family::R12000, mhz: 360, rev: (3, 5), l2_kb: 2048 }; +pub const R12000_400: Part = Part { mhz: 400, ..R12000_360 }; +pub const R14000_600: Part = Part { family: Family::R14000, mhz: 600, rev: (2, 4), l2_kb: 2048 }; + +/// Every part, for the bare-metal bench rig, which takes any. A name matches +/// the first listed, so the rig runs the later silicon of a clock that has two. +pub const ALL: &[&Part] = &[ + &R4400_150, &R5000_180_EARLY, &R5000_180, &R10000_195, &R10000_195_EARLY, &R10000_250, + &R12000_360, &R12000_400, &R14000_600, +]; + +impl Part { + /// Implementation, then major and minor revision. + pub const fn prid(&self) -> u32 { + self.family.implementation() << 8 | (self.rev.0 as u32) << 4 | self.rev.1 as u32 + } + + /// CP0 Count ticks at half the pipeline clock. + pub const fn count_hz(&self) -> u64 { + self.mhz as u64 * 500_000 + } + + pub fn is_r10000_family(&self) -> bool { + !matches!(self.family, Family::R4400 | Family::R5000) + } + + /// `r10000_195`, as `cpu = ` takes it. + pub fn name(&self) -> String { + format!("r{}_{}", self.family as u32, self.mhz) + } + + /// "MIPS R10000 195 MHz". + pub fn label(&self) -> String { + format!("MIPS R{} {} MHz", self.family as u32, self.mhz) + } +} + +/// The part in `cpus` that `name` means: its full name, or its family +/// (`r10000`), which is the first of that family listed. +pub fn find(cpus: &[&'static Part], name: &str) -> Option<&'static Part> { + let name = name.trim().to_ascii_lowercase(); + cpus.iter().copied().find(|p| name == p.name() || name == format!("r{}", p.family as u32)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The PRId is implementation, major, minor: the Indy's R5000 is rev 1.0 + /// and the O2's 2.1. A family name takes the first part of it a machine + /// lists. + #[test] + fn prid_is_implementation_then_revision_and_a_family_name_is_the_first_listed() { + assert_eq!((R5000_180_EARLY.prid(), R5000_180.prid()), (0x2310, 0x2321)); + assert_eq!(find(&[&R5000_180, &R10000_250, &R12000_400], "r10000"), Some(&R10000_250)); + } +} diff --git a/src/display_stop_tests.rs b/src/display_stop_tests.rs index 65954bf7f..35f0ac0d7 100644 --- a/src/display_stop_tests.rs +++ b/src/display_stop_tests.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}; use std::sync::{mpsc, Arc}; use std::time::{Duration, Instant}; -use crate::config::{CpuModel, GraphicsBoard, MachineConfig, MachineProfile}; +use crate::config::{GraphicsBoard, MachineConfig, MachineProfile}; use crate::dev::ng1::rex3::Renderer; use crate::workstation::Workstation; @@ -69,11 +69,12 @@ pub(crate) fn scratch(test: &str) -> PathBuf { } pub(crate) fn config(dir: &Path, profile: MachineProfile, board: GraphicsBoard) -> MachineConfig { - let (mb, cpu) = match profile { - MachineProfile::OctaneIp30 => (64, CpuModel::R10000), - MachineProfile::O2Ip32 => (32, CpuModel::R5000), - _ => (16, CpuModel::default()), + let mb = match profile { + MachineProfile::OctaneIp30 => 64, + MachineProfile::O2Ip32 => 32, + _ => 16, }; + let cpu = profile.cpus()[0]; let mut cfg = crate::bench_runner::bench_config([mb, 0, 0, 0], cpu); cfg.headless = false; cfg.machine.profile = profile; diff --git a/src/exit_persistence_tests.rs b/src/exit_persistence_tests.rs index 3723f9680..622e229ca 100644 --- a/src/exit_persistence_tests.rs +++ b/src/exit_persistence_tests.rs @@ -40,7 +40,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::Ordering; use std::sync::{Mutex, MutexGuard}; -use crate::config::{CpuModel, MachineConfig, MachineProfile, ScsiDeviceConfig}; +use crate::config::{MachineConfig, MachineProfile, ScsiDeviceConfig}; use crate::scsi::{scsi_cmd, ScsiBus, ScsiDataLength, ScsiRequest}; use crate::workstation::{self, Workstation}; @@ -203,7 +203,7 @@ enum Machine { } fn config(machine: Machine, dir: &Dir, ci: bool) -> MachineConfig { - let mut cfg = crate::bench_runner::bench_config([32, 0, 0, 0], CpuModel::default()); + let mut cfg = crate::bench_runner::bench_config([32, 0, 0, 0], &crate::cpu::parts::R4400_150); cfg.machine.profile = match machine { Machine::Indy => MachineProfile::IndyIp24, Machine::Indigo2 => MachineProfile::Indigo2Ip22, @@ -211,11 +211,11 @@ fn config(machine: Machine, dir: &Dir, ci: bool) -> MachineConfig { Machine::O2 => MachineProfile::O2Ip32, Machine::Octane => MachineProfile::OctaneIp30, }; - cfg.machine.cpu = match machine { - Machine::Indy | Machine::Indigo2 => CpuModel::R4400, - Machine::O2 => CpuModel::R5000, - Machine::Indigo2Ip28 | Machine::Octane => CpuModel::R10000, - }; + cfg.machine.cpu = Some(match machine { + Machine::Indy | Machine::Indigo2 => "r4400", + Machine::O2 => "r5000", + Machine::Indigo2Ip28 | Machine::Octane => "r10000", + }.into()); // The O2 and the Octane have no embedded PROM; an empty one builds. match machine { Machine::O2 => { diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 3aacdb09c..679f23b8d 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -15,8 +15,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use crate::config::{CpuModel, MachineConfig, MachineProfile}; -use crate::cpu::mips_cache_shadow::R10000ShadowCache; +use crate::config::{MachineConfig, MachineProfile}; +use crate::cpu::parts::{self, Part}; +use crate::cpu::mips_cache_v2::MipsCache; use crate::cpu::smp::World; use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; use crate::cpu::mips_tlb::MipsTlb; @@ -46,12 +47,9 @@ use crate::workstation::{MachineRef, MachineSlot, Workstation}; /// The PROM image tried when `prom` is left at the shared default, named /// as IRIX ships it in /usr/cpu/firmware. The Octane has no embedded PROM. pub const OCTANE_PROM_FILE: &str = "IP30prom.bin"; -/// The 195 MHz R10000 of SGI's standard 128 MB Octane configurations (the -/// OCTANE Technical Report): Count at half that. -pub const OCTANE_COUNT_HZ: u64 = 97_500_000; -/// R10000 revision 2.7, as an Octane's hinv reports it; the raw PRId is -/// inferred from the R10000 manual's encoding. -pub const OCTANE_PRID: u32 = 0x0927; +/// The processors an Octane shipped with, the default first. +pub const CPUS: &[&Part] = &[&parts::R10000_195, &parts::R12000_360, &parts::R14000_600]; + /// The Crossbow on the Octane reports revision code 4. const XBOW_REVISION: u32 = 4; /// The Octane2's Crossbow: revision code 5, which IRIX's inventory names @@ -160,7 +158,7 @@ pub struct OctaneMachine { impl OctaneMachine { pub fn new(cfg: &MachineConfig) -> Result { - MachineProfile::OctaneIp30.accepts_cpu(cfg.machine.cpu)?; + let part = MachineProfile::OctaneIp30.part(cfg.machine.cpu.as_deref())?; // `prom`, or IRIX's file name when `prom` is the shared default and // that file is not there: there is no embedded Octane PROM. let prom_path = MachineProfile::OctaneIp30.prom_path(&cfg.prom); @@ -353,7 +351,7 @@ impl OctaneMachine { if has_vpro { kind.label() } else { board.map_or("no graphics", |b| b.label()) } ); - let count_hz = cfg.clock.fixed_mhz.map_or(OCTANE_COUNT_HZ, |mhz| (mhz * 1_000_000.0) as u64); + let count_hz = cfg.clock.count_hz(part); let flash_lines: Vec<_> = (0..cpu_count) .map(|_| { let bus: Arc = phys.phys.clone(); @@ -365,7 +363,10 @@ impl OctaneMachine { ticks.extend((1..cpu_count).map(|_| Arc::new(AtomicU64::new(0)))); let memory = Arc::new(crate::ip30::memory_map(&cfg.banks[..4])); let cpus: Vec> = (0..cpu_count) - .map(|id| Self::build_cpu(id, cpu_count, memory.clone(), phys.phys.clone(), flash_lines[id as usize].clone(), &timer_manager, count_hz.max(1), ticks[id as usize].clone())) + .map(|id| { + let (sysad, cached, tm, fasttick) = (phys.phys.clone(), flash_lines[id as usize].clone(), &timer_manager, ticks[id as usize].clone()); + crate::with_cpu!(part, true, false, C => Self::build_cpu::(id, cpu_count, memory.clone(), sysad, cached, tm, count_hz.max(1), fasttick, part.prid())) + }) .collect(); let cpu = cpus[0].clone(); world.set_slots(cpus.iter().map(|c| c.park_slot()).collect()); @@ -575,8 +576,9 @@ impl OctaneMachine { } } + /// An Octane's processor `id` of `cpu_count` over cache type `C`. #[allow(clippy::too_many_arguments)] - fn build_cpu( + fn build_cpu( id: u8, cpu_count: u8, memory: Arc, @@ -585,17 +587,20 @@ impl OctaneMachine { tm: &Arc, count_hz: u64, fasttick: Arc, - ) -> Arc { - let cfg = MipsCpuConfig::for_model::(memory); + prid: u32, + ) -> Arc + where + C: crate::cpu::mips_cache_v2::CpuModel + MipsCache + From> + 'static, + MipsCpu: CpuDevice, + { + let cfg = MipsCpuConfig::for_model::(memory).with_prid(prid); let tlb = MipsTlb::new(cfg.tlb_entries); - let mut exec: MipsExecutor = MipsExecutor::new(sysad, tlb, &cfg); + let mut exec: MipsExecutor = MipsExecutor::new(sysad, tlb, &cfg); // Cached accesses see the PROM's dirty flash lines; see `CachedFlashLines`. - exec.cache = R10000ShadowCache::from(cached as Arc); + exec.cache = C::from(cached as Arc); exec.core.set_timer_manager(tm.clone()); exec.core.set_count_hz(count_hz); exec.core.fasttick_count = fasttick; - exec.core.reset_prid = OCTANE_PRID; - exec.core.cp0_prid = OCTANE_PRID; // IRIX zeroes Count and arms Compare a few instructions later; see // `MipsCore::count_write_samples`. exec.core.count_write_samples = true; @@ -605,7 +610,7 @@ impl OctaneMachine { exec.phys_mask = (1 << 40) - 1; exec.core.cpu_id = id; exec.core.smp = cpu_count > 1; - exec.batch_hook = Some(Box::new(move |exec: &mut MipsExecutor| { + exec.batch_hook = Some(Box::new(move |exec: &mut MipsExecutor| { LAST_PC[id as usize].store(exec.core.pc, Ordering::Relaxed); })); Arc::new(MipsCpu::new(exec)) @@ -1310,7 +1315,7 @@ mod snapshot_tests { fn machine_with(prom: &str, dir: &std::path::Path, edit: impl FnOnce(&mut MachineConfig)) -> OctaneMachine { let mut cfg = MachineConfig::default(); cfg.machine.profile = crate::config::MachineProfile::OctaneIp30; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.banks = [256, 0, 0, 0, 0, 0, 0, 0]; cfg.prom = prom.into(); cfg.nvram = dir.join("nvram.bin").to_string_lossy().into_owned(); diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index f78019b8b..1eb5c95b2 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -24,10 +24,10 @@ use std::time::{Duration, Instant}; use parking_lot::Mutex; -use crate::config::{CpuModel, MachineConfig, MachineProfile}; +use crate::config::{MachineConfig, MachineProfile}; +use crate::cpu::parts::{self, Part}; use crate::cpu::mips_core::CyclesPtr; use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; -use crate::cpu::mips_cache_shadow::{R10000O2ShadowCache, R5000ShadowCache}; use crate::cpu::mips_cache_v2::MipsCache; use crate::cpu::mips_tlb::MipsTlb; use crate::dev::crime_mem::O2Memory; @@ -154,11 +154,12 @@ const SNAPSHOT_MACHINE: &str = "o2_ip32"; /// release's, named as SGI shipped it. The O2 has no embedded PROM. pub const O2_PROM_FILE: &str = "ip32prom.rev4.18.bin"; -/// CP0 Count when `[clock] fixed_mhz` is unset: half of a 180 MHz CPU. -pub const O2_COUNT_MHZ: f64 = 90.0; +/// The processors an O2 shipped with, the default first. +pub const CPUS: &[&Part] = &[&parts::R5000_180, &parts::R10000_250, &parts::R12000_400]; impl O2Machine { pub fn new(cfg: &MachineConfig) -> Result { + let part = MachineProfile::O2Ip32.part(cfg.machine.cpu.as_deref())?; let o2 = &cfg.o2; // `prom`, or the last release's file when `prom` is the shared default // and that file is not there: there is no embedded O2 PROM. @@ -233,17 +234,13 @@ impl O2Machine { let timer_manager = Arc::new(TimerManager::new()); let sysad: Arc = bus.phys.clone(); let arcs_fpu = o2.arcs_fpu; - // `[clock] fixed_mhz` is CP0 Count; the CPU clock the PROM measures - // and reports is twice it. - let count_mhz = cfg.clock.fixed_mhz.unwrap_or(O2_COUNT_MHZ); - let cpu_mhz = (count_mhz * 2.0).round().max(1.0) as u32; + // The CPU clock the PROM measures and reports is twice Count. + let count_hz = cfg.clock.count_hz(part); + let cpu_mhz = (count_hz as f64 * 2.0 / 1e6).round().max(1.0) as u32; let boot = Arc::new(BootState::new()); let memory = Arc::new(crate::ip32::memory_map()); - let cpu: Arc = match cfg.machine.cpu { - CpuModel::R10000 => Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, cpu_mhz, boot.clone(), bus.ram.clone()), - CpuModel::R5000 => Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, cpu_mhz, boot.clone(), bus.ram.clone()), - CpuModel::R4400 => return Err("an O2 has an R5000 or an R10000 ([machine] cpu)".into()), - }; + let cpu: Arc = crate::with_cpu!(part, true, true, C => + Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, count_hz, boot.clone(), bus.ram.clone(), part.prid())); // ppmem: the CPU's cache and the JIT reach RAM through the memory's // window, publishing the mapped bitmap into the CPU's own fields, as @@ -363,19 +360,19 @@ impl O2Machine { }) } - fn build_cpu(memory: Arc, sysad: Arc, tm: &Arc, arcs_fpu: bool, mhz: u32, boot: Arc, - ram: Arc) -> Arc + fn build_cpu(memory: Arc, sysad: Arc, tm: &Arc, arcs_fpu: bool, count_hz: u64, boot: Arc, + ram: Arc, prid: u32) -> Arc where C: crate::cpu::mips_cache_v2::CpuModel + MipsCache + From> + 'static, MipsCpu: CpuDevice, { - let cfg = MipsCpuConfig::for_model::(memory); + let cfg = MipsCpuConfig::for_model::(memory).with_prid(prid); let tlb = MipsTlb::new(cfg.tlb_entries); let mut exec: MipsExecutor = MipsExecutor::new(sysad, tlb, &cfg); exec.cache.set_pd_tag_store_hook(Box::new(move |tag| ram.stack_tag_store(tag))); exec.core.set_timer_manager(tm.clone()); // Count runs at half the pipeline clock on both CPUs. - exec.core.set_count_hz(mhz.max(1) as u64 * 500_000); + exec.core.set_count_hz(count_hz.max(1)); // The no-ECC alias of memory reaches memory, not the PCI window the // bus decodes at the same low 32 bits. exec.phys_alias = Some((NO_ECC_BASE, RAM_WINDOW_SIZE as u64, RAM_BASE as u64)); diff --git a/src/machine.rs b/src/machine.rs index d63a74731..74f73c831 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -20,14 +20,17 @@ use crate::physical::RamBank; use crate::dev::prom::Prom; use crate::dev::mc::MemoryController; -/// Count on the IP28's 195 MHz R10000: half the pipeline clock, and what -/// IRIX assumes there (see where the MC is created). -const IP28_COUNT_HZ: u64 = 97_500_000; +/// The processors an Indy or `indigo2_ip22` shipped with, the default first. +/// The R10000 Indigo2 is the IP28 board. +pub const CPUS: &[&Part] = &[&parts::R4400_150, &parts::R5000_180_EARLY]; + +/// The IP28's. +pub const IP28_CPUS: &[&Part] = &[&parts::R10000_195_EARLY]; use crate::cpu::mips_tlb::MipsTlb; use crate::cpu::mips_exec::{MipsExecutor, MipsCpu, MipsCpuConfig, MipsCpuDebugAdapter}; use crate::gdb_stub::CpuDebug; -use crate::cpu::mips_cache_v2::{MipsCache, R4400Cache, R5000Cache}; -use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R4400ShadowCache, R5000ShadowCache}; +use crate::cpu::parts::{self, Part}; +use crate::cpu::mips_cache_v2::MipsCache; use crate::dev::hpc3::Hpc3; use crate::dev::ioc::{Ioc, GioSlot, GIO_SLOT_MAP, profile_idx}; use crate::monitor::Monitor; @@ -319,9 +322,11 @@ impl Machine { /// console and exit code delivered in-process instead of to stdout and /// `process::exit` (`crate::bench_runner`, `TestDevice::new_embedded`). pub fn new_with_testdev( - cfg: MachineConfig, + mut cfg: MachineConfig, testdev_override: Option>, ) -> Self { + // The rig takes any part: `MachineConfig::cpu_part` looks at this. + cfg.test_device |= testdev_override.is_some(); // Capture config flags that are needed after the local `cfg` binding // is shadowed later in this function. let ci_enabled = cfg.ci; @@ -333,8 +338,7 @@ impl Machine { let display_resolution = cfg.graphics.resolution; let newport_active = !cfg.headless && cfg.graphics_board() == GraphicsBoard::Newport; let monitor_port = cfg.monitor_port.unwrap_or(crate::monitor::DEFAULT_PORT); - let clock_fixed_mhz = cfg.clock.fixed_mhz; - let cfg_cpu_model = cfg.machine.cpu; + let cfg_cpu_model = cfg.cpu_part().expect("validate() accepted the cpu"); let guinness = cfg.machine.profile.guinness(); @@ -352,11 +356,7 @@ impl Machine { // 256 MB L2 and IRIX spends the boot flushing a cache that does not exist. // Keyed on the configured CPU, not on a cargo feature: the model is a // runtime choice, so a feature gate here silently stopped firing. - let model_has_l2 = match cfg_cpu_model { - crate::config::CpuModel::R4400 => ::L2_SIZE > 0, - crate::config::CpuModel::R5000 => ::L2_SIZE > 0, - crate::config::CpuModel::R10000 => ::L2_SIZE > 0, - }; + let model_has_l2 = cfg_cpu_model.l2_kb > 0; if !model_has_l2 { eeprom_mc.lock().set_cachsz(0); } @@ -373,12 +373,8 @@ impl Machine { // divider it programs. The generic 33 MHz Count and 50 MHz MC made // UST run at a third of real time (Quake in slow motion) and the // cycle counter at a quarter. - let ip28_count_hz: Option = if cfg.machine.profile.ip28() { - Some(clock_fixed_mhz.map_or(IP28_COUNT_HZ, |mhz| (mhz * 1_000_000.0) as u64)) - } else { - None - }; - if let Some(count_hz) = ip28_count_hz { + let count_hz = cfg.clock.count_hz(cfg_cpu_model); + if cfg.machine.profile.ip28() { // The R10000's Count ticks at half the pipeline clock. mc.set_clock_hz(2 * count_hz); } @@ -740,11 +736,11 @@ impl Machine { } // 5. CPU config + TLB + Executor. The model is a runtime choice, but each - // arm below monomorphises its own CPU — no per-model branch on the hot path. + // family monomorphises its own CPU (`crate::cpu::build`) — no per-model branch on the hot path. let sysad: Arc = phys.clone(); let memory = Arc::new(crate::ip22_bus::memory_map(cfg.machine.profile.ip28())); macro_rules! build_cpu { ($cache:ty) => {{ - let cfg = MipsCpuConfig::for_model::<$cache>(memory.clone()); + let cfg = MipsCpuConfig::for_model::<$cache>(memory.clone()).with_prid(cfg_cpu_model.prid()); let tlb = MipsTlb::new(cfg.tlb_entries); let mut executor: MipsExecutor = MipsExecutor::new(sysad.clone(), tlb, &cfg); // The MC decodes 32 physical address bits; see `MipsExecutor::phys_mask`. @@ -761,14 +757,10 @@ impl Machine { } } - // CP0 Count runs at a fixed frequency: DEFAULT_COUNT_HZ unless the - // user overrode it via `[clock] fixed_mhz` or the CLI. Must happen + // CP0 Count runs at half the chip's clock unless the user overrode + // it via `[clock] fixed_mhz` or the CLI. Must happen // before the core starts executing. - if let Some(hz) = ip28_count_hz { - executor.core.set_count_hz(hz); - } else if let Some(mhz) = clock_fixed_mhz { - executor.core.set_count_hz((mhz * 1_000_000.0) as u64); - } + executor.core.set_count_hz(count_hz); // Inject the shared fasttick_count Arc into the executor core before wrapping in MipsCpu. // (cycles has no equivalent here — it's inline on MipsCore.hot now.) @@ -791,18 +783,8 @@ impl Machine { }}} let shadow = cfg.machine.cache.is_shadow(cfg_cpu_model); - let cpu: Arc = match cfg_cpu_model { - // R4400/R5000: the functional caches, or the shadow ones that - // keep only what CACHE instructions observe — see CacheModel. - crate::config::CpuModel::R4400 if shadow => build_cpu!(R4400ShadowCache), - crate::config::CpuModel::R5000 if shadow => build_cpu!(R5000ShadowCache), - crate::config::CpuModel::R4400 => build_cpu!(R4400Cache), - crate::config::CpuModel::R5000 => build_cpu!(R5000Cache), - // IP28 uses the shadow cache: out of the data path entirely, with - // tag and data arrays that exist only to answer CACHE ops and the - // PROM's diagnostics. See mips_cache_shadow.rs. - crate::config::CpuModel::R10000 => build_cpu!(R10000ShadowCache), - }; + let cpu: Arc = + crate::with_cpu!(cfg_cpu_model, shadow, false, C => build_cpu!(C)); // The developer status bar shows the CPU's Count rate. status.set_count_hz(cpu.count_hz_atomic()); @@ -2248,7 +2230,7 @@ mod controller_lifetime_tests { std::thread::Builder::new() .stack_size(64 * 1024 * 1024) .spawn(|| Box::new(Machine::new(crate::bench_runner::bench_config( - [16, 0, 0, 0], crate::config::CpuModel::default())))) + [16, 0, 0, 0], &parts::R4400_150)))) .unwrap() .join() .unwrap() @@ -2301,7 +2283,7 @@ mod controller_lifetime_tests { let mut machine = std::thread::Builder::new() .stack_size(64 * 1024 * 1024) .spawn(move || { - let mut cfg = crate::bench_runner::bench_config([16, 0, 0, 0], crate::config::CpuModel::default()); + let mut cfg = crate::bench_runner::bench_config([16, 0, 0, 0], &parts::R4400_150); cfg.scsi.insert(1, crate::config::ScsiDeviceConfig { path: path_str, overlay: true, ..Default::default() }); Box::new(Machine::new(cfg)) }) @@ -2372,7 +2354,7 @@ mod stop_order_tests { // A REX3 saves its shape corpus when it stops; keep it out of $HOME. let dir = crate::display_stop_tests::scratch("indy-order"); let mut machine = big_stack(|| { - let mut cfg = crate::bench_runner::bench_config([16, 0, 0, 0], crate::config::CpuModel::default()); + let mut cfg = crate::bench_runner::bench_config([16, 0, 0, 0], &parts::R4400_150); cfg.headless = false; Box::new(Machine::new(cfg)) }); @@ -2405,7 +2387,7 @@ mod virtio_tests { cfg.headless = true; cfg.machine.profile = profile; if profile == MachineProfile::Indigo2Ip28 { - cfg.machine.cpu = crate::config::CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.banks = [512, 512, 0, 0, 0, 0, 0, 0]; cfg.graphics.board = Some(crate::config::GraphicsBoard::SolidImpact); } diff --git a/src/platform_profile_tests.rs b/src/platform_profile_tests.rs index f499e86c4..96286b863 100644 --- a/src/platform_profile_tests.rs +++ b/src/platform_profile_tests.rs @@ -42,10 +42,6 @@ mod tests { fn virtio_cfg(profile: MachineProfile) -> MachineConfig { let mut cfg = minimal_cfg(); cfg.machine.profile = profile; - // The Octane refuses the default R4400. - if !profile.cpus().contains(&cfg.machine.cpu) { - cfg.machine.cpu = profile.cpus()[0]; - } cfg.virtio_serial.enabled = true; cfg } diff --git a/src/validate_config.rs b/src/validate_config.rs index d6b6869f2..e73c0c975 100644 --- a/src/validate_config.rs +++ b/src/validate_config.rs @@ -149,7 +149,7 @@ pub fn report(path: &str, cfg: &MachineConfig) -> i32 { #[cfg(test)] mod tests { use super::*; - use crate::config::{banks_from4, CpuModel}; + use crate::config::banks_from4; fn dir(name: &str) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!("iris-check-{name}-{}", std::process::id())); @@ -216,7 +216,7 @@ mod tests { let d = dir("bad"); let mut cfg = MachineConfig::default(); cfg.machine.profile = MachineProfile::OctaneIp30; - cfg.machine.cpu = CpuModel::R4400; + cfg.machine.cpu = Some("r4400".into()); cfg.banks = banks_from4([128, 0, 0, 0]); cfg.prom = d.join("no-prom").to_string_lossy().into_owned(); cfg.scsi.clear(); @@ -226,7 +226,7 @@ mod tests { cfg.monitor_port = Some(8880); let all = problems(&cfg).join("\n"); for want in [ - "an Octane has an R10000", + "SGI Octane (IP30) takes [machine] cpu = ", "prom ", "scsi.1 ", "scsi.2: a disk needs a path", @@ -253,7 +253,7 @@ mod tests { assert!(warnings(&cfg)[0].contains("built-in PROM")); assert_eq!(report("indy.toml", &cfg), 0); cfg.machine.profile = MachineProfile::OctaneIp30; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.banks = banks_from4([128, 0, 0, 0]); assert!(warnings(&cfg).is_empty()); assert!(problems(&cfg).iter().any(|p| p.starts_with("prom "))); @@ -266,7 +266,7 @@ mod tests { let d = dir("lenient"); let mut cfg = MachineConfig::default(); cfg.machine.profile = MachineProfile::OctaneIp30; - cfg.machine.cpu = CpuModel::R10000; + cfg.machine.cpu = Some("r10000".into()); cfg.banks = banks_from4([128, 0, 0, 0]); cfg.prom = d.join("absent-prom").to_string_lossy().into_owned(); assert!(problems(&cfg).iter().any(|p| p.starts_with("prom ")), "{:?}", problems(&cfg)); From ee425b881acf239009d35cfaffc196fa29b9b9ec Mon Sep 17 00:00:00 2001 From: James Braid Date: Sat, 7 Nov 2026 07:10:00 -0800 Subject: [PATCH 6/7] cpu: serve cached accesses from lines staged by Index Store Tag and Data A boot PROM can stage a secondary-cache line with Index Store Tag (S) and Index Store Data (S) and then load, store or fetch from it before memory works. The R10000-family shadow cache kept those arrays for diagnostics only, so such an access read the bus. Per the NEC VR10000 manual, a valid tag whose PA[39:18] matches the access is a secondary hit that supplies the staged data, with no bus read. The shadow arrays already hold the staged tags and data, so the hit is a lookup in them: both ways at the access's index, valid state (TagLo[11:10] != 0), matching tag. A store to a staged line stays in the line and makes it DirtyExclusive. Index Writeback Invalidate, Hit Invalidate and Hit Writeback Invalidate (S) end the line, writing a DirtyExclusive one back; so does Index Store Tag. Snapshots carry the staged lines. No access tests for a staged line. While the cache holds any, it takes RAM off its window paths instead: read, write and the compare-and-swap paths see no mapped region, the decode cache gets no window, and the bus path they all fall to goes through a StagedBus that answers for the staged lines. The interpreter's and the JIT's inline loads and stores reach RAM through the micro-TLB and the window without calling the cache, and the JIT compiles from memory, so the executor also runs no compiled code until the last line ends. A guest that never stages a line runs the code it ran before; one that does runs in the interpreter on the bus path while it holds lines, and the shipped PROMs end every line they stage before the kernel runs. The R10000 translates every CACHE operand but Cache Barrier's, a mapped one taking TLB Refill or Invalid, and the secondary cache takes its set and way from the operand's physical address, not its virtual one. The word order of a refill is not settled by the manuals. The TagLo word lands at the lower address, which is inferred from software that stages lines this way and is marked as such. Replacement, coherence with other CPUs and DMA, and ECC are not modelled; the note in clanker/cpu lists the limits. The shadow model applies to the whole R10000 family on the R10000's tag layout, which is an inference for the R12000 and R14000. --- clanker/INDEX.md | 1 + clanker/cpu/staged-secondary-cache-lines.md | 62 + src/cpu/jitv2/equiv_test.rs | 222 +++ src/cpu/mips_cache_shadow.rs | 1356 ++++++++++++++++++- src/cpu/mips_cache_v2.rs | 26 +- src/cpu/mips_exec.rs | 54 +- 6 files changed, 1681 insertions(+), 40 deletions(-) create mode 100644 clanker/cpu/staged-secondary-cache-lines.md diff --git a/clanker/INDEX.md b/clanker/INDEX.md index ea8842f15..baaee2749 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -43,6 +43,7 @@ CPU architecture: CP0, TLB and its fast paths, caches, FPU, LL/SC, CPU models. B | [r5000-taglo-is-the-r4400-pstate.md](cpu/r5000-taglo-is-the-r4400-pstate.md) | rule | R5000 L1 TagLo uses the R4400 PState encoding | | [r5k-l1i-cache-bugs.md](cpu/r5k-l1i-cache-bugs.md) | open | R5000 secondary cache: no working configuration currently exists | | [sc-succeeds-on-llbit-alone.md](cpu/sc-succeeds-on-llbit-alone.md) | rule | `SC` succeeds on the LLbit alone — `LLAddr` is not a gate | +| [staged-secondary-cache-lines.md](cpu/staged-secondary-cache-lines.md) | rule | Staged secondary-cache lines hit until an explicit op ends them; the page fast paths must be told | | [tcache-discards-dirty-data-semantics.md](cpu/tcache-discards-dirty-data-semantics.md) | rule | tcache makes L1-D write-through, so Hit_Invalidate cannot discard | | [tcache-l2-instr-coherency.md](cpu/tcache-l2-instr-coherency.md) | rule | tcache: L2 decoded-instruction coherency, and how it was broken | | [tlb-c-field-uses-the-models-table.md](cpu/tlb-c-field-uses-the-models-table.md) | rule | The TLB decodes EntryLo C with the CPU model's table; C=4 read as uncached sent every Octane user access to the bus | diff --git a/clanker/cpu/staged-secondary-cache-lines.md b/clanker/cpu/staged-secondary-cache-lines.md new file mode 100644 index 000000000..acdacac84 --- /dev/null +++ b/clanker/cpu/staged-secondary-cache-lines.md @@ -0,0 +1,62 @@ +# Staged secondary-cache lines: a hit until an explicit operation ends them + +The R10000-family shadow cache (`src/cpu/mips_cache_shadow.rs`) serves a cached +load, store or fetch from a secondary line the guest staged with Index Store Tag +(S) and Index Store Data (S): a valid state in TagLo[11:10] and a tag equal to +PA[39:18] of the access. The rules are the R10000 cache index operations datasheet's +datasheet. A guest that never stages a line pays nothing: no access is tested +for one. + +## The limit: no replacement + +An ordinary cached miss never fills the shadow, so it never evicts. A staged +line stays until Index Writeback Invalidate, Hit Invalidate, Hit Writeback +Invalidate, an Index Store Tag that overwrites it, or power-on. A guest that +reuses the memory without invalidating reads the stale line and its stores stay +in the line, where hardware would have evicted it, writing a dirty one back to +the tag's address. + +Staged lines left at the login prompt, `l2 probe 0` in the monitor, CPU 0: +Octane (IP30 PROM, IRIX 6.5.30) 0, O2 R10000 (IRIX 6.5.30) 0. The shipped PROMs invalidate what they stage, so the machine is +back on the window paths and the JIT before IRIX runs. + +## How staged lines reach the data path + +No access is checked against them. While the cache holds any, it takes RAM +off its own fast paths instead (`serve_window`): + +- `read`, `write` and the compare-and-swap paths stop using ppmem's window + (`window_regions`), so every cached access takes the bus path, which goes + through a `StagedBus`: staged lines answer for their addresses, memory for + the rest; +- the decode cache gets no window (`fetch_window`), so it reads every word + through the same route; +- the executor runs no compiled code (`staged_lines_changed`). The JIT's inline + loads and stores reach RAM through the micro-TLB and ppmem's window without + calling the cache, and it compiles from memory. The executor learns of the + change from `take_staged_changed`, asked after each CACHE instruction. + +The last line ending undoes all three. While lines are held the whole machine +runs in the interpreter on the bus path, which is slow; the shipped PROMs end +what they stage before the kernel runs. + +Dropping the executor's part fails `a_compiled_load_hits_a_staged_line_not_the_window`, +`a_compiled_store_to_a_staged_line_stays_in_the_line` or +`a_staged_line_is_fetched_as_code_instead_of_the_compiled_page`; those run under +`jitv2` and under `lightning,jitv2`. + +## Inferred or absent + +- The TagLo word of a staged doubleword lands at the lower address + (`STAGED_TAGLO_AT_LOWER_ADDRESS`). Inferred from software, not measured. +- A store to a staged line makes it DirtyExclusive. NEC U10278EJ4V0UM p.246 + counts such requests; the datasheet does not state the transition. +- Every CACHE operand is translated first (exceptions: TLB Refill/Invalid, privilege Address Error, never TLB Modified). The secondary set, way and quadword come from the operand's physical address, which the executor passes in place of the virtual one, the primaries' from its virtual address. An operand on an uncached page is undefined on the R10000 and goes ahead as if cached. +- The datasheet clears the R10000. The R12000 and R14000 use the same + tag layout here, which is an inference. +- Not modelled: predecode, primary-cache staged hits, ECC and check bits, the + virtual-index purge, the BadVAddr/Context/EntryHi values after a CACHE-raised TLB exception, Status.CH, the + tag rewrite on invalidate (only the state clears), coherence with other CPUs + and DMA. +- Staged lines are saved in snapshots (`save_cache_state`). The rest of the + shadow arrays, which only diagnostics read, are not. diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 8a24f3d76..ee4a0d651 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -4440,6 +4440,228 @@ mod tests { assert_ne!(interp.gpr[2] >> 32, 0x1122_3344, "R5000 sign-extends"); } + /// A secondary line staged with Index Store Tag and Index Store Data is + /// fetched as code, not the page the JIT compiled from memory: staging it + /// drops the page from the JIT (the micro-TLB flush in `exec_cache`, then + /// `jitv2_track_pcp` declining the page), and invalidating the line puts + /// the page back on memory and the JIT. Production dispatch, inline + /// compile, an R10000-family shadow cache. + #[test] + fn a_staged_line_is_fetched_as_code_instead_of_the_compiled_page() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + use crate::cpu::mips_isa::{OP_ADDIU, OP_CACHE}; + let pc = 0xFFFF_FFFF_8000_1000u64; + let ra = 0xFFFF_FFFF_8000_2000u64; + let addiu = |imm: u32| make_i(OP_ADDIU, 0, 2, imm as u16); + let jr = make_r(crate::cpu::mips_isa::OP_SPECIAL, 31, 0, 0, 0, crate::cpu::mips_isa::FUNCT_JR); + let mem = Arc::new(MockMemory::new()); + for (i, w) in [addiu(0x11), jr, 0].into_iter().enumerate() { + mem.set_word(0x1000 + 4 * i as u64, w); + } + let bus: Arc = mem.clone(); + let mut exec = Box::new(MipsExecutor::::new( + bus, PassthroughTlb::default(), &MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))))); + exec.core.cp0_config = (exec.core.cp0_config & !7) | 3; // KSEG0 cacheable + exec.install_jit_hooks(); + exec.jitv2_inline_compile = true; + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + let run = |exec: &mut MipsExecutor| -> u64 { + exec.core.pc = pc; + exec.core.gpr[31] = ra; + exec.core.gpr[2] = 0; + for _ in 0..16 { + if exec.core.pc == ra { break; } + exec.step_jit(); + } + assert_eq!(exec.core.pc, ra, "the code returns"); + exec.core.gpr[2] + }; + let cache_op = |exec: &mut MipsExecutor, op: u32, lo: u32, hi: u32| { + exec.core.gpr[4] = pc; + exec.core.cp0_taglo = lo as u64; + exec.core.cp0_taghi = hi; + assert_eq!(exec.exec(make_i(OP_CACHE, 4, op, 0)), crate::cpu::mips_exec::EXEC_COMPLETE); + }; + + // From memory, until the page is compiled and entered. + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x11); + } + assert!(!exec.pcp.is_null(), "the page is on the JIT"); + assert!(unsafe { (*exec.pcp).is_published(0) }, "and compiled"); + + // Stage the line at PA 0x1000: valid CleanExclusive, tag PA[39:18] = 0. + cache_op(&mut exec, 0x0b, 2 << 10, 0); + assert!(exec.pcp.is_null(), "staging dropped the page from the JIT"); + // TagLo is the word at the lower address (the inferred refill order). + cache_op(&mut exec, 0x1f, addiu(0x22), jr); + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x22, "the staged word runs, not the compiled page"); + } + assert!(exec.pcp.is_null(), "and the page stays off it"); + + // Invalidate the line (Index Writeback Invalidate): memory and the JIT again. + cache_op(&mut exec, 0x03, 0, 0); + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x11); + } + assert!(!exec.pcp.is_null(), "the page is back on the JIT"); + } + + /// The data side: compiled loads reach RAM through the micro-TLB and + /// ppmem's window, never the cache. Warm the micro-TLB on a page, stage a + /// secondary line in it, and the compiled load returns the staged word; + /// invalidating the line sends it back to the window. + #[test] + fn a_compiled_load_hits_a_staged_line_not_the_window() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + use crate::cpu::mips_cache_v2::MipsCache; + use crate::cpu::mips_isa::{OP_CACHE, OP_LW}; + let pc = 0xFFFF_FFFF_8000_1000u64; + let ra = 0xFFFF_FFFF_8000_8000u64; + let data_va = 0xFFFF_FFFF_8000_2000u64; // PA 0x2000 + let lw = make_i(OP_LW, 4, 2, 0); + let jr = make_r(crate::cpu::mips_isa::OP_SPECIAL, 31, 0, 0, 0, crate::cpu::mips_isa::FUNCT_JR); + let mem = Arc::new(MockMemory::new()); + for (i, w) in [lw, jr, 0].into_iter().enumerate() { + mem.set_word(0x1000 + 4 * i as u64, w); + } + let bus: Arc = mem.clone(); + let mut exec = Box::new(MipsExecutor::::new( + bus, PassthroughTlb::default(), &MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))))); + exec.core.cp0_config = (exec.core.cp0_config & !7) | 3; // KSEG0 cacheable + // Out of reset ERL is set and the micro-TLB declines to fill; this test + // is about what it holds. + exec.core.cp0_status &= !crate::cpu::mips_core::STATUS_ERL; + exec.resync_privilege_state(); + // A window over the first 64 KiB of RAM, region 0 mapped, as ppmem + // publishes them: u32 words in native order. + let mut window = vec![0u8; 0x10000]; + unsafe { *(window.as_mut_ptr().add(0x2000) as *mut u32) = 0x0a0a_0a0a }; + let mut gens: Vec = (0..16).map(|_| AtomicU64::new(0)).collect(); + unsafe { + exec.set_tcache_window(window.as_mut_ptr()); + exec.set_tcache_gen_window(gens.as_mut_ptr()); + *exec.tcache_bitmap_ptr() = 1; + } + exec.core.ppmem_bitmap = 1; + exec.install_jit_hooks(); + assert!(exec.cache.jit_dc_geometry().supported, "the inline path is on"); + exec.jitv2_inline_compile = true; + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + let run = |exec: &mut MipsExecutor| -> u64 { + exec.core.pc = pc; + exec.core.gpr[31] = ra; + exec.core.gpr[4] = data_va; + exec.core.gpr[2] = 0; + for _ in 0..16 { + if exec.core.pc == ra { break; } + exec.step_jit(); + } + assert_eq!(exec.core.pc, ra, "the code returns"); + exec.core.gpr[2] + }; + let cache_op = |exec: &mut MipsExecutor, op: u32, lo: u32, hi: u32| { + exec.core.gpr[5] = data_va; + exec.core.cp0_taglo = lo as u64; + exec.core.cp0_taghi = hi; + assert_eq!(exec.exec(make_i(OP_CACHE, 5, op, 0)), crate::cpu::mips_exec::EXEC_COMPLETE); + }; + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x0a0a_0a0a, "from the window"); + } + // PA 0x2000: valid CleanExclusive, tag PA[39:18] = 0. TagLo is the + // word at the lower address (the inferred refill order). + cache_op(&mut exec, 0x0b, 2 << 10, 0); + cache_op(&mut exec, 0x1f, 0x5a5a_5a5a, 0x1111_1111); + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x5a5a_5a5a, "from the staged line"); + } + cache_op(&mut exec, 0x03, 0, 0); + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x0a0a_0a0a, "from the window again"); + } + } + + /// The store side: a compiled store through a warm write micro-TLB entry + /// goes inline into ppmem's window. Once the page holds a staged line the + /// store lands in the line instead: the window and memory do not change, + /// a cached load sees the value, and invalidating the line (it is + /// DirtyExclusive after the store) writes the value back to memory. + #[test] + fn a_compiled_store_to_a_staged_line_stays_in_the_line() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + use crate::cpu::mips_cache_v2::MipsCache; + use crate::cpu::mips_isa::{OP_CACHE, OP_SW}; + let pc = 0xFFFF_FFFF_8000_1000u64; + let ra = 0xFFFF_FFFF_8000_8000u64; + let data_va = 0xFFFF_FFFF_8000_2000u64; // PA 0x2000 + let sw = make_i(OP_SW, 4, 5, 0); + let jr = make_r(crate::cpu::mips_isa::OP_SPECIAL, 31, 0, 0, 0, crate::cpu::mips_isa::FUNCT_JR); + let mem = Arc::new(MockMemory::new()); + for (i, w) in [sw, jr, 0].into_iter().enumerate() { + mem.set_word(0x1000 + 4 * i as u64, w); + } + let bus: Arc = mem.clone(); + let mut exec = Box::new(MipsExecutor::::new( + bus, PassthroughTlb::default(), &MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))))); + exec.core.cp0_config = (exec.core.cp0_config & !7) | 3; // KSEG0 cacheable + exec.core.cp0_status &= !crate::cpu::mips_core::STATUS_ERL; + exec.resync_privilege_state(); + let mut window = vec![0u8; 0x10000]; + let mut gens: Vec = (0..16).map(|_| AtomicU64::new(0)).collect(); + unsafe { + exec.set_tcache_window(window.as_mut_ptr()); + exec.set_tcache_gen_window(gens.as_mut_ptr()); + *exec.tcache_bitmap_ptr() = 1; + } + exec.core.ppmem_bitmap = 1; + exec.install_jit_hooks(); + assert!(exec.cache.jit_dc_geometry().supported, "the inline path is on"); + exec.jitv2_inline_compile = true; + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + let win = |window: &Vec| unsafe { *(window.as_ptr().add(0x2000) as *const u32) }; + let run = |exec: &mut MipsExecutor, v: u64| { + exec.core.pc = pc; + exec.core.gpr[31] = ra; + exec.core.gpr[4] = data_va; + exec.core.gpr[5] = v; + for _ in 0..16 { + if exec.core.pc == ra { break; } + exec.step_jit(); + } + assert_eq!(exec.core.pc, ra, "the code returns"); + }; + let cache_op = |exec: &mut MipsExecutor, op: u32, lo: u32, hi: u32| { + exec.core.gpr[6] = data_va; + exec.core.cp0_taglo = lo as u64; + exec.core.cp0_taghi = hi; + assert_eq!(exec.exec(make_i(OP_CACHE, 6, op, 0)), crate::cpu::mips_exec::EXEC_COMPLETE); + }; + for v in 1..=4u64 { + run(&mut exec, v); + assert_eq!(win(&window), v as u32, "into the window"); + } + cache_op(&mut exec, 0x0b, 2 << 10, 0); + cache_op(&mut exec, 0x1f, 0, 0); + for v in 0x10..0x14u64 { + run(&mut exec, v); + assert_eq!(win(&window), 4, "the window is left alone"); + assert_eq!(exec.cache.read::<4>(data_va, 0x2000).data, v, "the line has it"); + } + assert_eq!(mem.get_word(0x2000), 0, "and memory"); + cache_op(&mut exec, 0x03, 0, 0); + assert_eq!(mem.get_word(0x2000), 0x13, "written back when the line goes"); + run(&mut exec, 0x20); + assert_eq!(win(&window), 0x20, "back to the window"); + } + #[test] fn lwl_lwr_combine_to_a_full_unaligned_word_load_matches_interpreter() { // The canonical compiler-generated idiom for an unaligned word diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index c50605acc..84e13dc25 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -27,16 +27,23 @@ //! exception is the R10000's secondary tag, whose reserved bits read as zero //! (`CpuModel::SECONDARY_TAG_MASK`). //! -//! The shadow is deliberately not consulted by `read`/`write`/`fetch`. If it -//! ever needs to be, this type is the wrong shape and should say so loudly -//! rather than growing a slow path. +//! The shadow is not consulted by `read`/`write`/`fetch`, with one exception +//! that costs a normal access nothing: on the R10000 family, a +//! secondary line the guest *staged* — a valid tag written by Index Store Tag +//! and data by Index Store Data — is a real hit for later cached loads, stores +//! and fetches, served from the shadow without a bus access. See [`Staged`]. +//! Nothing else about the shadow reaches the data path: a line the guest never +//! staged is memory, as before, and ordinary misses never fill or evict. So a +//! staged line lives until an explicit index or hit operation, an Index Store +//! Tag or power-on ends it; a guest that reuses the memory without +//! invalidating reads the stale line, where hardware would have evicted it. use std::cell::UnsafeCell; use std::sync::Arc; use crate::cpu::mips_cache_v2::{ cache_op_name, model, r4k_load_taglo, r4k_store_taglo, CpuModel, FetchInstrResult, MipsCache, - C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, + C_HINV, C_HWBINV, C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, }; use crate::traits::{BusDevice, BusRead64, BUS_OK}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_OP, devlog_is_active, devlog_mask}; @@ -92,6 +99,166 @@ const MRU_BIT: u64 = 1 << 63; /// R4400 and R5000 index differently — see `tag_slot`. const WAYS: usize = 2; +/// The secondary tag fields a cached access is compared against. The tag +/// register pair holds PA[35:18] in TagLo[31:14] and PA[39:36] in TagHi[3:0], +/// which the assembled 64-bit tag keeps at bits 31:14 and 35:32. NEC +/// U10278EJ4V0UM pp.267-270; the R10000 cache index operations datasheet, "Tag +/// operations". +const L2_TAG_PA_MASK: u64 = 0xf_ffff_c000; + +/// Where the secondary state field sits in the assembled tag: TagLo[11:10], +/// 0 Invalid, 1 Shared, 2 CleanExclusive, 3 DirtyExclusive. +const L2_STATE_SHIFT: u32 = 10; +const L2_STATE_DIRTY_EXCLUSIVE: u64 = 3; + +/// Does the secondary tag hold a valid line? Every state but Invalid does. +#[inline(always)] +fn l2_tag_valid(tag: u64) -> bool { + (tag >> L2_STATE_SHIFT) & 3 != 0 +} + +/// The tag bits a physical address is compared with. +#[inline(always)] +fn l2_tag_of(pa: u64) -> u64 { + (((pa >> 18) & 0x3_ffff) << 14) | (((pa >> 36) & 0xf) << 32) +} + +/// **Inferred, not established.** Which word of a staged doubleword a word +/// access at the lower address sees. A secondary quadword's doubleword is +/// staged as TagHi (bits 63:32) and TagLo (bits 31:0), but neither NEC manual +/// says how those halves become the words of a primary refill. The only +/// evidence is software that stages a line this way and then fetches it +/// (the R10000 cache index operations datasheet, "Not established"): it is consistent +/// only with the TagLo word landing at the lower address, and that is a +/// simulation, not a measurement. `false` is the manual-literal order. Index +/// Load Data is unaffected; it returns the registers as they were stored. +const STAGED_TAGLO_AT_LOWER_ADDRESS: bool = true; + +/// A staged doubleword as a big-endian memory doubleword: what a `ld` at its +/// address returns. The conversion is its own inverse. +#[inline(always)] +fn staged_to_memory(d: u64) -> u64 { + if STAGED_TAGLO_AT_LOWER_ADDRESS { d.rotate_left(32) } else { d } +} + +/// The lines a guest has made valid, and so what a cached access must +/// consult before memory. +/// +/// A line is staged when its tag is valid; its data are whatever Index Store +/// Data put in the data shadow, so the shadow arrays *are* the staged lines and +/// this holds only their count. Accesses are not checked against it. While the +/// count is nonzero the cache instead takes RAM off its fast paths +/// (`serve_window`): `read`, `write` and `fetch` stop using ppmem's window, +/// so every cached access takes the bus path, and only that path looks for a +/// staged line. The executor keeps compiled code off as well +/// (`MipsCache::holds_staged_lines`): the JIT's inline loads and stores reach +/// the window without calling the cache, and its code was built from memory. +/// A guest that never stages a line pays nothing. One that does runs in the +/// interpreter, on the bus path, until the last line ends; the shipped PROMs +/// end every line they stage before the kernel runs. +struct Staged { + /// Secondary and primary I tags with a valid state. + valid: u32, + /// `valid` became or stopped being zero since `take_staged_changed`. + changed: bool, +} + +/// The bus as a cached access sees it while staged lines are held: a staged +/// secondary line answers for its addresses, memory for the rest. Put in +/// front of the bus by `ShadowCache::serve_window`, and dropped with the last +/// line. +struct StagedBus< + const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, + const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, +> { + /// The cache that installed this and outlives it: it holds this only in + /// its own `route`, and is not moved while the CPU runs. + cache: *const ShadowCache, + bus: Arc, +} + +// Safety: reached only from the CPU thread, through the cache that owns it. +unsafe impl< + const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, + const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, + > Send + for StagedBus +{ +} +unsafe impl< + const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, + const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, + > Sync + for StagedBus +{ +} + +impl< + const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, + const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, + > StagedBus +{ + fn cache(&self) -> &ShadowCache { + unsafe { &*self.cache } + } +} + +impl< + const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, + const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, + const FIR: u32, const TLB_ENTRIES: usize, const MODEL: u8, + > BusDevice + for StagedBus +{ + fn read8(&self, addr: u64) -> crate::traits::BusRead8 { + match self.cache().staged_read::<1>(addr) { + Some(v) => crate::traits::BusRead8::ok(v as u8), + None => self.bus.read8(addr), + } + } + fn read16(&self, addr: u64) -> crate::traits::BusRead16 { + match self.cache().staged_read::<2>(addr) { + Some(v) => crate::traits::BusRead16::ok(v as u16), + None => self.bus.read16(addr), + } + } + fn read32(&self, addr: u64) -> crate::traits::BusRead32 { + match self.cache().staged_read::<4>(addr) { + Some(v) => crate::traits::BusRead32::ok(v as u32), + None => self.bus.read32(addr), + } + } + fn read64(&self, addr: u64) -> BusRead64 { + match self.cache().staged_read::<8>(addr) { + Some(v) => BusRead64::ok(v), + None => self.bus.read64(addr), + } + } + fn write8(&self, addr: u64, val: u8) -> u32 { + if self.cache().staged_write::<1>(addr, val as u64) { BUS_OK } else { self.bus.write8(addr, val) } + } + fn write16(&self, addr: u64, val: u16) -> u32 { + if self.cache().staged_write::<2>(addr, val as u64) { BUS_OK } else { self.bus.write16(addr, val) } + } + fn write32(&self, addr: u64, val: u32) -> u32 { + if self.cache().staged_write::<4>(addr, val as u64) { BUS_OK } else { self.bus.write32(addr, val) } + } + fn write64(&self, addr: u64, val: u64) -> u32 { + if self.cache().staged_write::<8>(addr, val) { BUS_OK } else { self.bus.write64(addr, val) } + } + /// A store-conditional to a staged line is an ordinary store into it. + fn cas32(&self, addr: u64, old: u32, new: u32) -> Option { + if self.cache().staged_slot(addr).is_some() { None } else { self.bus.cas32(addr, old, new) } + } + fn cas64(&self, addr: u64, old: u64, new: u64) -> Option { + if self.cache().staged_slot(addr).is_some() { None } else { self.bus.cas64(addr, old, new) } + } +} + /// Is the shadow cache's operation trace on, at this level of detail? /// /// `log l2c mask op` traces the tag operations a diagnostic actually cares @@ -132,6 +299,12 @@ pub struct ShadowCache< const MODEL: u8, > { downstream: Arc, + /// Where a cached access that the window does not serve goes, and where a + /// fetch's decode cache reads words it cannot take from the window: + /// `downstream` itself, or while staged lines are held a [`StagedBus`] in + /// front of it (`serve_window`). The bus paths make no test for staged + /// lines; the object they call is what changes. + route: UnsafeCell>, llbit: UnsafeCell, lladdr: UnsafeCell, /// CP0 ECC on the way in to a store, and on the way out of a load. @@ -149,6 +322,13 @@ pub struct ShadowCache< tc_bitmap: UnsafeCell, /// The window's `ppmem::window_shift`, set before the CPU runs. tc_shift: UnsafeCell, + /// How many of the bitmap's 64 regions `read` and `write` may take from + /// the window, and the window `fetch` gives the decode cache: all of it, + /// or none and null while the guest holds staged lines (`serve_window`). + /// R10000 family only: other models read the window whenever the bitmap + /// maps it. + window_regions: UnsafeCell, + fetch_window: UnsafeCell<*const u8>, #[cfg(feature = "jitv2")] tc_gen: UnsafeCell<*mut std::sync::atomic::AtomicU64>, /// Decoded instructions, by physical page. Never touches the shadow @@ -156,6 +336,8 @@ pub struct ShadowCache< decode: UnsafeCell, /// See [`MipsCache::set_pd_tag_store_hook`]. Consulted by `cache_op` only. pd_tag_hook: UnsafeCell>>, + /// Secondary lines the guest staged. R10000 family only; empty otherwise. + staged: UnsafeCell, } // Safety: the CPU thread is the only accessor, as for every other cache model @@ -195,6 +377,7 @@ impl< pub fn new(downstream: Arc) -> Self { Self { + route: UnsafeCell::new(downstream.clone()), downstream, llbit: UnsafeCell::new(false), lladdr: UnsafeCell::new(0), @@ -209,20 +392,27 @@ impl< tc_base: UnsafeCell::new(std::ptr::null_mut()), tc_bitmap: UnsafeCell::new(0), tc_shift: UnsafeCell::new(crate::ppmem::BITMAP_SHIFT), + window_regions: UnsafeCell::new(64), + fetch_window: UnsafeCell::new(std::ptr::null()), #[cfg(feature = "jitv2")] tc_gen: UnsafeCell::new(std::ptr::null_mut()), decode: UnsafeCell::new(crate::cpu::decode_cache::DecodeCache::new()), pd_tag_hook: UnsafeCell::new(None), + staged: UnsafeCell::new(Staged { valid: 0, changed: false }), } } /// ppmem's window base if `phys` is RAM the window maps, as the region /// bitmap says; `None` for the PROM, MMIO and the low-512KB alias, which go /// through the bus. The bitmap stays zero until the window is wired, so a - /// set bit also means the base is valid. + /// set bit also means the base is valid. On the R10000 family, `None` for + /// everything while staged lines are held (`window_regions`): the bound is + /// read from memory instead of being 64, which costs no instruction. #[inline(always)] fn ram(&self, phys: u64) -> Option<*mut u8> { - if crate::ppmem::bitmap_maps_at(unsafe { *self.tc_bitmap.get() }, unsafe { *self.tc_shift.get() }, phys) { + let regions = if Self::R10K { unsafe { *self.window_regions.get() } } else { 64 }; + let region = phys >> unsafe { *self.tc_shift.get() }; + if region < regions && unsafe { *self.tc_bitmap.get() } & (1u64 << region) != 0 { Some(unsafe { *self.tc_base.get() }) } else { None @@ -341,6 +531,230 @@ impl< s.ecc[slot]); } } + + // ---- Staged secondary lines --------------------------------------- + // + // Everything below is R10000-family only and runs only once a guest has + // made a secondary tag valid. The rules are those of the R10000 cache index + // operations datasheet: a cached access that misses the primary + // caches looks up both secondary ways at the access's index, and a way + // whose tag holds the access's physical address in a valid state supplies + // the data. Index Store Data (S) wrote a whole quadword, so the other + // doubleword of it reads as zero rather than as memory. + // + // Not modelled, and loudly absent rather than guessed: + // - the primary caches. A primary line the guest staged with the primary + // index operations is not consulted, so where one overlaps a staged + // secondary line the secondary data win, not the primary's; + // - replacement. An ordinary cached miss never fills, so never evicts: + // a staged line lives until an index or hit operation, or an Index + // Store Tag, ends it (`clanker/cpu/staged-secondary-cache-lines.md`); + // - tag ECC and data check bits. Nothing is checked when a line hits, so + // a line staged with bad check bits hits without a Cache Error; + // - the virtual-index field, TagLo[8:7]. A mismatch with the access's + // VA[13:12] purges primary lines at the recorded index, and there are + // no primary lines here to purge; + // - a CACHE operand on an uncached page. It is undefined on the R10000; + // the operation goes ahead as if the page were cached; + // - the BadVAddr, Context and EntryHi values after a CACHE-raised TLB + // exception, which the manuals do not give; + // - coherence. Another CPU's or a DMA device's write to memory a CPU + // holds in a staged line does not reach the line, and the line's data + // are not visible to them; + // - the part. The datasheet clears the R10000 only. The R12000 and R14000 + // use the same tag layout and rules here, which is an inference + // ("R12000 behavior needs separate evidence"); both keep only the bits + // of `CpuModel::SECONDARY_TAG_MASK`; + // - undefined encodings. Operations with op[1:0] = 10 (`CACH_SI`) are + // undefined on the R10000 and do nothing. + + /// Are there any staged lines? Asked on the bus paths only: while there + /// are, every cached access takes one (`ram`, `fetch_window`). + #[inline(always)] + fn staged_any(&self) -> bool { + Self::R10K && unsafe { (*self.staged.get()).valid } != 0 + } + + /// The tag index and data-shadow slot of the doubleword at `pa` in + /// whichever way holds it as a valid line, if one does. Same layout as `data_slot`: a way's + /// doublewords are every `WAYS`-th slot, indexed by `PA[N-2:3]`. + fn staged_line(&self, pa: u64) -> Option<(usize, usize)> { + let l2 = unsafe { &*self.l2.get() }; + if l2.data.is_empty() { + return None; + } + let set = (pa as usize / L2_LINE) % (Self::L2_LINES / WAYS).max(1); + let want = l2_tag_of(pa); + (0..WAYS).find_map(|way| { + let tag = l2.tags[set * WAYS + way]; + (l2_tag_valid(tag) && (tag ^ want) & L2_TAG_PA_MASK == 0) + .then(|| (set * WAYS + way, (((pa as usize) >> 3) % (l2.data.len() / WAYS)) * WAYS + way)) + }) + } + + /// The data-shadow slot of the doubleword at `pa`, if a staged line holds it. + fn staged_slot(&self, pa: u64) -> Option { + self.staged_line(pa).map(|(_, slot)| slot) + } + + /// A cached load of `SIZE` bytes at `pa` from a staged line, as the + /// zero-extended value; `None` if no staged line holds it. + #[inline(never)] + fn staged_read(&self, pa: u64) -> Option { + let slot = self.staged_slot(pa)?; + let m = staged_to_memory(unsafe { (*self.l2.get()).data[slot] }); + let shift = 64 - 8 * SIZE as u32 - 8 * (pa & 7) as u32; + Some(if SIZE == 8 { m } else { (m >> shift) & ((1u64 << (8 * SIZE)) - 1) }) + } + + /// A cached store of `SIZE` bytes at `pa` into a staged line. The line + /// is a write-back cache's: the store stays in it and memory is not + /// written. Returns whether a staged line took the store. + #[inline(never)] + fn staged_write(&self, pa: u64, val: u64) -> bool { + let shift = 64 - 8 * SIZE as u32 - 8 * (pa & 7) as u32; + let mask = if SIZE == 8 { u64::MAX } else { ((1u64 << (8 * SIZE)) - 1) << shift }; + self.staged_write_masked(pa, val << shift, mask) + } + + /// `staged_write` for bytes selected by `mask` of the doubleword at `pa`, + /// `val` already in memory-doubleword position. + fn staged_write_masked(&self, pa: u64, val: u64, mask: u64) -> bool { + let Some((idx, slot)) = self.staged_line(pa) else { return false }; + let l2 = unsafe { &mut *self.l2.get() }; + let m = (staged_to_memory(l2.data[slot]) & !mask) | (val & mask); + l2.data[slot] = staged_to_memory(m); + // **Inferred.** A store to a CleanExclusive or Shared secondary block + // asks the secondary cache to make it DirtyExclusive (NEC + // U10278EJ4V0UM p.246, performance counter events 14 and 15). The + // datasheet does not state the transition, so this takes it as + // immediate. A later writeback invalidate then sends the stored bytes + // to memory instead of dropping them. + l2.tags[idx] |= L2_STATE_DIRTY_EXCLUSIVE << L2_STATE_SHIFT; + true + } + + /// A tag store changed whether a line is staged: it was (`was`) and is + /// (`now`). The first line staged and the last one ending switch the + /// cache's fast paths off and on again (`serve_window`). + fn staged_move(&self, was: bool, now: bool) { + let st = unsafe { &mut *self.staged.get() }; + let before = st.valid != 0; + st.valid = st.valid - was as u32 + now as u32; + if (st.valid != 0) != before { + self.serve_window(!before); + } + } + + /// Take RAM off the window paths while lines are staged (`hold`), and put + /// the staged lines in front of the bus paths; or undo both. The fast + /// paths read the fields this sets anyway, so the switch costs a normal + /// access nothing. The decode cache forgets every page: its slots hold + /// window pointers, or, after staging, words read through the bus that a + /// line may since have ended. + #[cold] + #[inline(never)] + fn serve_window(&self, hold: bool) { + unsafe { + *self.window_regions.get() = if hold { 0 } else { 64 }; + *self.fetch_window.get() = if hold { std::ptr::null() } else { *self.tc_base.get() }; + *self.route.get() = if hold { + Arc::new(StagedBus { cache: self as *const Self, bus: self.downstream.clone() }) + } else { + self.downstream.clone() + }; + (*self.staged.get()).changed = true; + (*self.decode.get()).flush(); + } + } + + /// Store a secondary tag, keeping the staged-line count in step. + fn set_l2_tag(&self, idx: usize, new: u64) { + let l2 = unsafe { &mut *self.l2.get() }; + let old = l2.tags[idx]; + l2.tags[idx] = new; + let (was, now) = (l2_tag_valid(old), l2_tag_valid(new)); + if was || now { + self.staged_move(was, now); + } + } + + /// Physical address of the line a secondary `tag` at `set` names: the + /// tag's PA[39:18] and the set's own PA[17:B] below it. + fn l2_line_pa(&self, tag: u64, set: usize) -> u64 { + (((tag >> 14) & 0x3_ffff) << 18) + | (((tag >> 32) & 0xf) << 36) + | ((set as u64 * L2_LINE as u64) & 0x3_ffff) + } + + /// Invalidate the secondary line at tag slot `idx`: a DirtyExclusive one + /// is first written back to the system interface, then its state goes to + /// Invalid (NEC U12754EJ1V0UMJ1 pp.19, 25, 27). The tag keeps its other + /// fields. The manual also rewrites the tag with the operand's physical + /// address and generates its ECC; neither is modelled, because the + /// shadow stores tags verbatim and has no ECC. + fn l2_invalidate(&self, idx: usize, writeback: bool) { + let l2 = unsafe { &*self.l2.get() }; + let tag = l2.tags[idx]; + if !l2_tag_valid(tag) { + return; + } + if writeback && (tag >> L2_STATE_SHIFT) & 3 == L2_STATE_DIRTY_EXCLUSIVE { + self.l2_write_line_back(idx); + } + self.set_l2_tag(idx, tag & !(3 << L2_STATE_SHIFT)); + } + + /// Send the block at tag slot `idx` to the system interface, doubleword by + /// doubleword in memory order, at the address its tag names. + fn l2_write_line_back(&self, idx: usize) { + let l2 = unsafe { &*self.l2.get() }; + if l2.data.is_empty() { + return; + } + let (set, way) = (idx / WAYS, idx % WAYS); + let base = self.l2_line_pa(l2.tags[idx], set); + let per_way = l2.data.len() / WAYS; + for i in 0..(L2_LINE / 8) { + let slot = ((set * L2_LINE / 8 + i) % per_way) * WAYS + way; + // A bus error on the way out has nowhere to go. + let _ = self.downstream.write64(base + 8 * i as u64, staged_to_memory(l2.data[slot])); + } + } + + /// Hit Invalidate / Hit Writeback Invalidate (S): act on whichever way + /// holds `pa`. A miss does nothing. Status.CH is not modelled. + fn l2_hit_invalidate(&self, pa: u64, writeback: bool) { + let set = (pa as usize / L2_LINE) % (Self::L2_LINES / WAYS).max(1); + let want = l2_tag_of(pa); + for way in 0..WAYS { + let idx = set * WAYS + way; + let tag = unsafe { (*self.l2.get()).tags[idx] }; + if l2_tag_valid(tag) && (tag ^ want) & L2_TAG_PA_MASK == 0 { + self.l2_invalidate(idx, writeback); + } + } + } + + /// Test hook: does a valid staged line lie in the page holding `phys`? + #[cfg(test)] + pub fn page_staged(&self, phys: u64) -> bool { + let l2 = unsafe { &*self.l2.get() }; + l2.tags.iter().enumerate().any(|(i, &t)| l2_tag_valid(t) && self.l2_line_pa(t, i / WAYS) >> 12 == phys >> 12) + } + + /// Index Writeback Invalidate, Hit Invalidate and Hit Writeback Invalidate + /// (S) while lines are staged. Other caches' forms do nothing here. + #[cold] + #[inline(never)] + fn end_staged_line(&self, op: u32, sel: u32, virt_addr: u64, phys_addr: u64) { + match (op, sel) { + (C_IWBINV, CACH_SD) => self.l2_invalidate(self.tag_slot(sel, virt_addr), true), + (C_HINV, CACH_SD) => self.l2_hit_invalidate(phys_addr, false), + (C_HWBINV, CACH_SD) => self.l2_hit_invalidate(phys_addr, true), + _ => {} + } + } } impl< @@ -403,7 +817,12 @@ impl< // ppmem's window, in every build: memory is this cache's only store, and // reading RAM through the window is one host load instead of a bus call. unsafe fn set_tcache_window(&self, base: *mut u8) { - unsafe { *self.tc_base.get() = base }; + unsafe { + *self.tc_base.get() = base; + if !self.staged_any() { + *self.fetch_window.get() = base; + } + } } fn set_tcache_shift(&self, shift: u32) { @@ -425,6 +844,20 @@ impl< /// nothing a guest can see. fn fetch_is_pure(&self) -> bool { true } + fn holds_staged_lines(&self) -> bool { + self.staged_any() + } + + fn take_staged_changed(&self) -> bool { + let changed = unsafe { &mut (*self.staged.get()).changed }; + if Self::R10K && *changed { + *changed = false; + true + } else { + false + } + } + #[cfg(feature = "jitv2")] fn tcache_gen_ptr(&self) -> *mut u8 { unsafe { *self.tc_gen.get() as *mut u8 } } @@ -454,11 +887,15 @@ impl< fn fetch(&self, _virt_addr: u64, phys_addr: u64) -> FetchInstrResult { // Decoded instructions by physical page — see `decode_cache`. let dc = unsafe { &mut *self.decode.get() }; + // Only a word the decode cache cannot take from the window comes + // here. While lines are staged that is every word (`fetch_window`), + // and the route serves the staged lines (`StagedBus`). let word = |a: u64| { - let r = self.downstream.read32(a); + let r = unsafe { &*self.route.get() }.read32(a); if r.is_ok() { Ok(r.data) } else { Err(r.status) } }; - let (window, bitmap) = unsafe { (*self.tc_base.get() as *const u8, *self.tc_bitmap.get()) }; + let window = if Self::R10K { unsafe { *self.fetch_window.get() } } else { unsafe { *self.tc_base.get() } }; + let bitmap = unsafe { *self.tc_bitmap.get() }; match dc.fetch(phys_addr, window, bitmap, IC_LINE, word) { Ok(p) => FetchInstrResult::hit(p), Err(status) => FetchInstrResult::exception(status), @@ -487,17 +924,20 @@ impl< }; return BusRead64::ok(data); } + // Every cached access comes here while lines are staged (`ram`), and + // the route serves them (`StagedBus`). + let bus = unsafe { &*self.route.get() }; if SIZE == 1 { - let r = self.downstream.read8(phys_addr); + let r = bus.read8(phys_addr); BusRead64 { status: r.status, data: r.data as u64 } } else if SIZE == 2 { - let r = self.downstream.read16(phys_addr); + let r = bus.read16(phys_addr); BusRead64 { status: r.status, data: r.data as u64 } } else if SIZE == 4 { - let r = self.downstream.read32(phys_addr); + let r = bus.read32(phys_addr); BusRead64 { status: r.status, data: r.data as u64 } } else { - self.downstream.read64(phys_addr) + bus.read64(phys_addr) } } @@ -521,14 +961,15 @@ impl< self.bump_gen(phys_addr); return BUS_OK; } + let bus = unsafe { &*self.route.get() }; if SIZE == 1 { - self.downstream.write8(phys_addr, val as u8) + bus.write8(phys_addr, val as u8) } else if SIZE == 2 { - self.downstream.write16(phys_addr, val as u16) + bus.write16(phys_addr, val as u16) } else if SIZE == 4 { - self.downstream.write32(phys_addr, val as u32) + bus.write32(phys_addr, val as u32) } else { - self.downstream.write64(phys_addr, val) + bus.write64(phys_addr, val) } } @@ -546,12 +987,13 @@ impl< self.bump_gen(phys_addr); return BUS_OK; } + let bus = unsafe { &*self.route.get() }; let aligned = phys_addr & !7; - let r = self.downstream.read64(aligned); + let r = bus.read64(aligned); if !r.is_ok() { return r.status; } - self.downstream.write64(aligned, (r.data & !mask) | (val & mask)) + bus.write64(aligned, (r.data & !mask) | (val & mask)) } /// The whole point of the type. @@ -579,6 +1021,16 @@ impl< cache_op_name(cache_op)); } + // op[1:0] = 10 is not defined for the R10000 (NEC U12754EJ1V0UMJ1 + // p.16): it does nothing, so it cannot stage or end a line. The + // operations that could touch a line test for it; the hit and + // invalidate operations act on `CACH_SD` only. + // + // On the R10000 the executor passes the secondary operations' physical + // address as `virt_addr`: the secondary set, way and quadword come from + // it, the primaries' from the virtual address (NEC U12754EJ1V0UMJ1 + // pp.14-15). + // R4400/R5000: a tag keeps only its fields, and reads back with // generated check bits — exactly what the full model returns. if !Self::R10K && matches!(op, C_IST | C_ILT) { @@ -599,6 +1051,7 @@ impl< // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim // unless the part sets a secondary mask: a tag test is a round trip, // and round trips do not require knowing what the bits mean. + C_IST if Self::R10K && sel == CACH_SI => 0, C_IST => { let idx = self.tag_slot(sel, virt_addr); let mask = if matches!(sel, CACH_SI | CACH_SD) { @@ -619,10 +1072,16 @@ impl< // here destroyed real tag bits. let set = (idx / WAYS).min(s.mru.len() - 1); s.mru[set] = phys_addr & MRU_BIT != 0; - s.tags[idx] = phys_addr & mask & !MRU_BIT; + let tag = phys_addr & mask & !MRU_BIT; + if Self::R10K && sel == CACH_SD { + self.set_l2_tag(idx, tag); + } else { + s.tags[idx] = tag; + } } 0 } + C_ILT if Self::R10K && sel == CACH_SI => 0, C_ILT => { let idx = self.tag_slot(sel, virt_addr); let s = self.shadow(sel); @@ -657,7 +1116,7 @@ impl< s.data[slot] } } - C_R10K_ISD if Self::R10K => { + C_R10K_ISD if Self::R10K && sel != CACH_SI => { self.index_store_data(sel, virt_addr, phys_addr); 0 } @@ -671,6 +1130,15 @@ impl< 0 } + // The operations that end a staged line. With none staged they + // have nothing to end, since a valid tag is a staged line, so they + // and every other cache's fall through: they move data between a + // cache and memory, and here those never disagree. + C_IWBINV | C_HINV | C_HWBINV if Self::R10K && self.staged_any() => { + self.end_staged_line(op, sel, virt_addr, phys_addr); + 0 + } + // Invalidate, writeback, fill, and the R4000 hit operations. All // no-ops: there is nothing held that could be stale or dirty. _ => 0, @@ -685,16 +1153,105 @@ impl< s.tags.fill(0); s.mru.fill(false); } + let st = unsafe { &mut *self.staged.get() }; + if st.valid != 0 { + st.valid = 0; + self.serve_window(false); + } unsafe { (*self.decode.get()).flush() }; } - /// A restore replaces RAM wholesale; drop every decoded instruction. The - /// shadow arrays are not snapshotted (nothing reads them but diagnostics). - fn load_cache_state(&self, _v: &toml::Value) -> Result<(), String> { + /// The staged secondary lines, which cached accesses read: tag, MRU bit, + /// and the line's doublewords with their check bits. The rest of the shadow + /// arrays is for diagnostics, which nothing reads back after a restore, and + /// is not saved. Empty when nothing is staged. + fn save_cache_state(&self) -> toml::Value { + let mut t = toml::value::Table::new(); + if !self.staged_any() { + return toml::Value::Table(t); + } + let l2 = unsafe { &*self.l2.get() }; + let per_way = l2.data.len() / WAYS; + let u = |v: u64| toml::Value::Array(vec![toml::Value::Integer((v & 0xffff_ffff) as i64), toml::Value::Integer((v >> 32) as i64)]); + let mut lines = Vec::new(); + for (idx, &tag) in l2.tags.iter().enumerate() { + if !l2_tag_valid(tag) { + continue; + } + let (set, way) = (idx / WAYS, idx % WAYS); + let slots = (0..L2_LINE / 8).map(|i| ((set * L2_LINE / 8 + i) % per_way) * WAYS + way); + let mut e = toml::value::Table::new(); + e.insert("idx".into(), toml::Value::Integer(idx as i64)); + e.insert("tag".into(), u(tag)); + e.insert("mru".into(), toml::Value::Boolean(l2.mru[set.min(l2.mru.len() - 1)])); + e.insert("data".into(), toml::Value::Array(slots.clone().map(|s| u(l2.data[s])).collect())); + e.insert("ecc".into(), toml::Value::Array(slots.map(|s| toml::Value::Integer(l2.ecc[s] as i64)).collect())); + lines.push(toml::Value::Table(e)); + } + t.insert("staged_l2".into(), toml::Value::Array(lines)); + toml::Value::Table(t) + } + + /// A restore replaces RAM wholesale: drop every decoded instruction, end + /// every staged line and bring back the saved ones. + fn load_cache_state(&self, v: &toml::Value) -> Result<(), String> { + let l2 = unsafe { &mut *self.l2.get() }; + if Self::R10K { + for idx in 0..l2.tags.len() { + if l2_tag_valid(l2.tags[idx]) { + self.set_l2_tag(idx, l2.tags[idx] & !(3 << L2_STATE_SHIFT)); + } + } + } + let lines = v.get("staged_l2").and_then(|l| l.as_array()); + let u = |v: &toml::Value| -> Result { + let a = v.as_array().filter(|a| a.len() == 2).ok_or("staged line: bad doubleword")?; + let w = |x: &toml::Value| x.as_integer().map(|i| i as u64 & 0xffff_ffff).ok_or("staged line: bad word"); + Ok(w(&a[0])? | w(&a[1])? << 32) + }; + for e in lines.into_iter().flatten().filter(|_| Self::R10K) { + let idx = e.get("idx").and_then(|i| i.as_integer()).ok_or("staged line: no index")? as usize; + if idx >= l2.tags.len() || l2.data.is_empty() { + return Err(format!("staged line index {idx} outside the secondary cache")); + } + let (set, way) = (idx / WAYS, idx % WAYS); + let per_way = l2.data.len() / WAYS; + let data = e.get("data").and_then(|d| d.as_array()).ok_or("staged line: no data")?; + let ecc = e.get("ecc").and_then(|d| d.as_array()).ok_or("staged line: no check bits")?; + if data.len() != L2_LINE / 8 || ecc.len() != data.len() { + return Err("staged line: wrong size for this cache".into()); + } + for (i, (d, c)) in data.iter().zip(ecc).enumerate() { + let slot = ((set * L2_LINE / 8 + i) % per_way) * WAYS + way; + l2.data[slot] = u(d)?; + l2.ecc[slot] = c.as_integer().ok_or("staged line: bad check bits")? as u32; + } + l2.mru[set.min(l2.mru.len() - 1)] = e.get("mru").and_then(|m| m.as_bool()).unwrap_or(false); + self.set_l2_tag(idx, u(e.get("tag").ok_or("staged line: no tag")?)?); + } unsafe { (*self.decode.get()).flush() }; Ok(()) } + /// Dirty staged lines go to memory, and stay staged. + fn writeback_all(&self) { + if !self.staged_any() { + return; + } + let l2 = unsafe { &*self.l2.get() }; + for idx in 0..l2.tags.len() { + if l2_tag_valid(l2.tags[idx]) && (l2.tags[idx] >> L2_STATE_SHIFT) & 3 == L2_STATE_DIRTY_EXCLUSIVE { + self.l2_write_line_back(idx); + } + } + } + + /// `l2 probe` in the monitor: how many secondary lines are staged. + fn debug_probe(&self, _cache_name: &str, _virt_addr: u64, _phys_addr: u64) -> String { + let st = unsafe { &*self.staged.get() }; + format!("staged secondary lines: {}", st.valid) + } + fn get_config(&self, cache_target: u32) -> (usize, usize) { match cache_target { CACH_PI => (IC_SIZE, IC_LINE), @@ -720,8 +1277,10 @@ impl< fn cas_ram32(&self, phys: u64, old: u32, new: u32) -> Option { use std::sync::atomic::{AtomicU32, Ordering}; // Memory beyond the window (the Octane's above 4 GB) is on the bus. + // A staged line is not in RAM: `StagedBus` has the caller store the + // ordinary way. let Some(base) = self.ram(phys) else { - return self.downstream.cas32(phys, old, new); + return unsafe { &*self.route.get() }.cas32(phys, old, new); }; let word = unsafe { AtomicU32::from_ptr(base.add(phys as usize) as *mut u32) }; let ok = word.compare_exchange(old, new, Ordering::AcqRel, Ordering::Relaxed).is_ok(); @@ -734,7 +1293,7 @@ impl< fn cas_ram64(&self, phys: u64, old: u64, new: u64) -> Option { use std::sync::atomic::{AtomicU64, Ordering}; let Some(base) = self.ram(phys) else { - return self.downstream.cas64(phys, old, new); + return unsafe { &*self.route.get() }.cas64(phys, old, new); }; let word = unsafe { AtomicU64::from_ptr(base.add(phys as usize) as *mut u64) }; let ok = word.compare_exchange(old.rotate_left(32), new.rotate_left(32), Ordering::AcqRel, Ordering::Relaxed).is_ok(); @@ -1120,6 +1679,751 @@ mod tests { check::("R5000", &[CACH_PI, CACH_PD]); } + // ---- Index-operation staging --------------------------------------- + // + // Rules are the R10000 cache index operations datasheet's; the section + // names are that sheet's. + + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::Mutex; + use crate::cpu::mips_cache_v2::{C_HINV, C_HWBINV, C_IWBINV}; + + /// Memory that reads 0xee everywhere, counts reads and records writes, so + /// a staged value can never be mistaken for it and a bus access shows. + #[derive(Default)] + struct Bus { + reads: AtomicU32, + writes: Mutex>, + } + + impl BusDevice for Bus { + fn read8(&self, _: u64) -> crate::traits::BusRead8 { + self.reads.fetch_add(1, Ordering::Relaxed); + crate::traits::BusRead8::ok(0xee) + } + fn read16(&self, _: u64) -> crate::traits::BusRead16 { + self.reads.fetch_add(1, Ordering::Relaxed); + crate::traits::BusRead16::ok(0xeeee) + } + fn read32(&self, _: u64) -> crate::traits::BusRead32 { + self.reads.fetch_add(1, Ordering::Relaxed); + crate::traits::BusRead32::ok(0xeeee_eeee) + } + fn read64(&self, _: u64) -> BusRead64 { + self.reads.fetch_add(1, Ordering::Relaxed); + BusRead64::ok(0xeeee_eeee_eeee_eeee) + } + fn write8(&self, a: u64, v: u8) -> u32 { self.writes.lock().unwrap().push((a, v as u64)); BUS_OK } + fn write16(&self, a: u64, v: u16) -> u32 { self.writes.lock().unwrap().push((a, v as u64)); BUS_OK } + fn write32(&self, a: u64, v: u32) -> u32 { self.writes.lock().unwrap().push((a, v as u64)); BUS_OK } + fn write64(&self, a: u64, v: u64) -> u32 { self.writes.lock().unwrap().push((a, v)); BUS_OK } + } + + /// An R10000 secondary cache (1 MiB, 128-byte lines, 4096 sets) over `Bus`. + fn staged_cache() -> (R10000ShadowCache, Arc) { + let bus = Arc::new(Bus::default()); + (R10000ShadowCache::from(bus.clone() as Arc), bus) + } + + /// A line in set 0x20 (PA[18:7]), tag PA[35:18] = 4. + const PA: u64 = 0x0010_1000; + const MEM: u64 = 0xeeee_eeee_eeee_eeee; + + /// The operand VA of an index operation on `pa`'s set, `way` in bit 0. + fn operand(pa: u64, way: u64) -> u64 { + 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8) | way + } + + /// Index Store Tag (S): the line of `pa`, in `way`, with `state`. + fn stage_tag(c: &R10000ShadowCache, pa: u64, way: u64, state: u64) { + c.cache_op(C_IST | CACH_SD, operand(pa & !127, way), l2_tag_of(pa) | state << L2_STATE_SHIFT); + } + + /// Index Store Data (S): the doubleword of `pa` becomes TagHi:TagLo = `d`. + fn stage_data(c: &R10000ShadowCache, pa: u64, way: u64, d: u64) { + c.cache_op(C_R10K_ISD | CACH_SD, operand(pa & !7, way), d); + } + + fn load(c: &R10000ShadowCache, pa: u64) -> u64 { + let r = c.read::(operand(pa, 0), pa); + assert!(r.is_ok()); + r.data + } + + /// Encoding: the 5-bit op field is `op[4:2]` over `cache[1:0]`, and + /// the secondary operations are the ones the sheet lists. ISUM pp.70-72. + #[test] + fn the_secondary_index_and_hit_operations_have_the_listed_encodings() { + assert_eq!(C_IWBINV | CACH_SD, 0x03); + assert_eq!(C_ILT | CACH_SD, 0x07); + assert_eq!(C_IST | CACH_SD, 0x0b); + assert_eq!(C_HINV | CACH_SD, 0x13); + assert_eq!(C_HWBINV | CACH_SD, 0x17); + assert_eq!(C_R10K_ILD | CACH_SD, 0x1b); + assert_eq!(C_R10K_ISD | CACH_SD, 0x1f); + // Primary data and instruction, for the neighbours a decoder mixes up. + assert_eq!((C_IST | CACH_PD, C_R10K_ISD | CACH_PD), (0x09, 0x1d)); + assert_eq!((C_IST | CACH_PI, C_R10K_ISD | CACH_PI), (0x08, 0x1c)); + assert_eq!(cache_op_name(0x0b), "Index_Store_Tag(SD)"); + } + + /// The whole thing through the CACHE instruction (major opcode 0x2f, op in + /// rt, base register and offset) and the interpreter's cached load: TagLo + /// and TagHi are what the operations store, and `lw` hits the line. + #[test] + fn cache_instructions_stage_a_line_a_cached_load_hits() { + use crate::cpu::mips_exec::{MipsCpuConfig, MipsExecutor, EXEC_COMPLETE}; + use crate::cpu::mips_isa::{OP_CACHE, OP_LW}; + use crate::cpu::mips_tlb::PassthroughTlb; + let bus = Arc::new(Bus::default()); + let mut exec = MipsExecutor::::new( + bus.clone() as Arc, PassthroughTlb::default(), + &MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false)))); + #[cfg(feature = "jitv2")] + { exec.jitv2_dispatch_enabled = false; } + exec.core.cp0_config = (exec.core.cp0_config & !7) | 3; // KSEG0: cacheable + let i = |op: u32, rs: u32, rt: u32, imm: u32| (op << 26) | (rs << 21) | (rt << 16) | imm; + exec.core.gpr[4] = operand(PA, 0); + let tag = l2_tag_of(PA) | 2 << L2_STATE_SHIFT; + exec.core.cp0_taglo = tag as u32 as u64; + exec.core.cp0_taghi = (tag >> 32) as u32; + assert_eq!(exec.exec(i(OP_CACHE, 4, 0x0b, 0)), EXEC_COMPLETE); + exec.core.cp0_taglo = 0x1111_2222u64; // word at the lower address (inferred) + exec.core.cp0_taghi = 0x3333_4444; + assert_eq!(exec.exec(i(OP_CACHE, 4, 0x1f, 0)), EXEC_COMPLETE); + assert_eq!(exec.exec(i(OP_LW, 4, 5, 0)), EXEC_COMPLETE); + assert_eq!(exec.core.gpr[5], 0x1111_2222); + assert_eq!(exec.exec(i(OP_LW, 4, 6, 4)), EXEC_COMPLETE); + assert_eq!(exec.core.gpr[6], 0x3333_4444); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "no system-bus read"); + } + + /// Way select (ISUM pp.71-72): address bit 0 picks the way for every + /// index operation, and the two ways of a set hold different lines. + #[test] + fn the_way_comes_from_address_bit_0() { + let (c, bus) = staged_cache(); + let other = PA + (1 << 19); // same set, tag PA[35:18] differs + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0xaaaa_aaaa_0000_0001); + stage_tag(&c, other, 1, 2); + stage_data(&c, other, 1, 0xbbbb_bbbb_0000_0002); + assert_eq!(load::<8>(&c, PA).rotate_left(32), 0xaaaa_aaaa_0000_0001); + assert_eq!(load::<8>(&c, other).rotate_left(32), 0xbbbb_bbbb_0000_0002); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0); + // The tags are read back from the way that was addressed. + let tag = |way| c.cache_op(C_ILT | CACH_SD, operand(PA, way), 0); + assert_eq!(tag(0), l2_tag_of(PA) | 2 << L2_STATE_SHIFT); + assert_eq!(tag(1), l2_tag_of(other) | 2 << L2_STATE_SHIFT); + // Ending way 1 leaves way 0. + stage_tag(&c, other, 1, 0); + assert_eq!(load::<8>(&c, other), MEM); + assert_ne!(load::<8>(&c, PA), MEM); + } + + /// Tag state (UM p.84): Shared, CleanExclusive and DirtyExclusive are + /// valid; Invalid misses. + #[test] + fn every_state_but_invalid_is_valid() { + for (state, hits) in [(0, false), (1, true), (2, true), (3, true)] { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, state); + stage_data(&c, PA, 0, 0x1234_5678_9abc_def0); + let got = load::<8>(&c, PA); + assert_eq!(got != MEM, hits, "state {state}"); + assert_eq!(bus.reads.load(Ordering::Relaxed), if hits { 0 } else { 1 }, "state {state}"); + } + } + + /// The hit after Index Store Tag + Index Store Data: the staged data, at + /// every access width, with no bus read; the other doubleword of the + /// quadword reads zero (UM p.91, ISUM p.30), not memory. + #[test] + fn a_staged_line_hits_with_its_data_and_no_bus_read() { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, 2); + // The word at the lower address is TagLo: the inferred order, below. + stage_data(&c, PA + 8, 0, 0x0102_0304_a1a2_a3a4); + let m = 0xa1a2_a3a4_0102_0304u64; // as `ld` at PA+8 sees it + assert_eq!(load::<8>(&c, PA + 8), m); + assert_eq!(load::<4>(&c, PA + 8), m >> 32); + assert_eq!(load::<4>(&c, PA + 12), m & 0xffff_ffff); + assert_eq!(load::<2>(&c, PA + 10), (m >> 32) & 0xffff); + assert_eq!(load::<1>(&c, PA + 11), (m >> 32) & 0xff); + assert_eq!(load::<1>(&c, PA + 15), m & 0xff); + assert_eq!(load::<8>(&c, PA), 0, "the quadword's other doubleword is zero"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0); + // Two Index Store Data to one quadword leave only the second. + stage_data(&c, PA, 0, 0x5555_5555_6666_6666); + assert_eq!(load::<8>(&c, PA + 8), 0); + assert_ne!(load::<8>(&c, PA), 0); + } + + /// A miss: the tag does not match, the state is invalid, or the access + /// is in another set. The access reads the bus and sees memory. + #[test] + fn a_tag_that_does_not_match_misses() { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0x1234_5678_9abc_def0); + for bit in 14..32 { + // TagLo[31:14] is PA[35:18]: a flipped bit is another address. + let pa = PA ^ (1 << (bit + 4)); + let before = bus.reads.load(Ordering::Relaxed); + assert_eq!(load::<8>(&c, pa), MEM, "PA bit {}", bit + 4); + assert_eq!(bus.reads.load(Ordering::Relaxed), before + 1); + } + // TagHi[3:0] is PA[39:36]. + let before = bus.reads.load(Ordering::Relaxed); + assert_eq!(load::<8>(&c, PA | 1 << 36), MEM); + assert_eq!(bus.reads.load(Ordering::Relaxed), before + 1); + // Another set, same tag. + assert_eq!(load::<8>(&c, PA + 128), MEM); + // The line itself still hits. + assert_ne!(load::<8>(&c, PA), MEM); + } + + /// **Inferred.** Staging TagHi = W1 and TagLo = W0 into one doubleword: + /// a word load at its lower address returns W0 (TagLo lands at the lower + /// address), the higher address W1, and a doubleword load has W0 on top. + /// The manual-literal order would return W1 first. Index Load Data + /// returns the registers as stored under either order. + #[test] + fn the_inferred_refill_order_puts_taglo_at_the_lower_address() { + let (c, _) = staged_cache(); + let (w0, w1) = (0x1111_1111u64, 0x2222_2222u64); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, w1 << 32 | w0); + assert!(STAGED_TAGLO_AT_LOWER_ADDRESS, "the inferred order is the one modelled"); + assert_eq!(load::<4>(&c, PA), w0, "lower address: TagLo"); + assert_eq!(load::<4>(&c, PA + 4), w1, "higher address: TagHi"); + assert_eq!(load::<8>(&c, PA), w0 << 32 | w1); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, operand(PA, 0), 0), w1 << 32 | w0, + "Index Load Data is independent of the order"); + // The conversion itself, for either order. + assert_eq!(staged_to_memory(w1 << 32 | w0), w0 << 32 | w1); + assert_eq!(staged_to_memory(staged_to_memory(7)), 7); + } + + /// A fetch hits too, and sees the line end when it is invalidated: the + /// decode cache does not keep serving the staged word. + #[test] + fn a_fetch_runs_the_staged_word_until_the_line_is_invalidated() { + let (c, bus) = staged_cache(); + let word = |c: &R10000ShadowCache| unsafe { + (*c.fetch(operand(PA, 0), PA).instr).raw + }; + assert_eq!(word(&c), 0xeeee_eeee, "memory before staging"); + let reads = bus.reads.load(Ordering::Relaxed); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0x2402_0022_0000_0000 | 0x0003_e008); // TagHi = addiu, TagLo = jr ra + assert_eq!(word(&c), 0x0003_e008, "TagLo is the word at the lower address"); + assert_eq!(word(&c), 0x0003_e008); + let f = c.fetch(operand(PA, 0) + 4, PA + 4); + assert_eq!(unsafe { (*f.instr).raw }, 0x2402_0022); + let f = c.fetch(operand(PA, 0) + 8, PA + 8); + assert_eq!(unsafe { (*f.instr).raw }, 0, "the quadword's pad is a nop"); + assert_eq!(bus.reads.load(Ordering::Relaxed), reads); + c.cache_op(C_IWBINV | CACH_SD, operand(PA, 0), 0); + assert_eq!(word(&c), 0xeeee_eeee, "back to memory"); + } + + /// Eviction and invalidation end the hit: Index Writeback Invalidate by + /// index, Hit Invalidate and Hit Writeback Invalidate by address, an + /// Index Store Tag that writes another tag or an invalid state, and + /// power-on. An operation on another line does not. + #[test] + fn invalidation_and_eviction_end_the_hit() { + let ends: [(&str, Box); 6] = [ + ("Index Writeback Invalidate", Box::new(|c| { c.cache_op(C_IWBINV | CACH_SD, operand(PA, 0), 0); })), + ("Hit Invalidate", Box::new(|c| { c.cache_op(C_HINV | CACH_SD, operand(PA, 0), PA); })), + ("Hit Writeback Invalidate", Box::new(|c| { c.cache_op(C_HWBINV | CACH_SD, operand(PA, 0), PA); })), + ("Index Store Tag, invalid", Box::new(|c| stage_tag(c, PA, 0, 0))), + ("Index Store Tag, another line", Box::new(|c| stage_tag(c, PA + (1 << 19), 0, 2))), + ("power-on", Box::new(|c| c.power_on())), + ]; + for (name, end) in ends { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0x1234_5678_9abc_def0); + assert_ne!(load::<8>(&c, PA), MEM, "{name}: hits before"); + end(&c); + assert_eq!(load::<8>(&c, PA), MEM, "{name}: misses after"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 1, "{name}"); + assert!(c.take_staged_changed(), "{name}"); + assert!(!c.page_staged(PA) || name == "Index Store Tag, another line"); + } + // Another line's invalidation, and a hit operation that misses, leave it. + let (c, _) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0x1234_5678_9abc_def0); + c.cache_op(C_IWBINV | CACH_SD, operand(PA + 128, 0), 0); + c.cache_op(C_HINV | CACH_SD, operand(PA, 0), PA + (1 << 19)); + c.cache_op(C_IWBINV | CACH_SD, operand(PA, 1), 0); // the other way + assert_ne!(load::<8>(&c, PA), MEM); + } + + /// Writeback (ISUM pp.18-19, 27): invalidating a DirtyExclusive line + /// sends its block to the system interface at the tag's address; other + /// states only notify. Staged data and memory order agree. + #[test] + fn a_dirty_exclusive_line_is_written_back_when_it_goes() { + for (state, written) in [(3, true), (2, false), (1, false)] { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, state); + stage_data(&c, PA + 16, 0, 0x1111_1111_2222_2222); + c.cache_op(C_IWBINV | CACH_SD, operand(PA, 0), 0); + let w = bus.writes.lock().unwrap().clone(); + assert_eq!(!w.is_empty(), written, "state {state}"); + if written { + assert_eq!(w.len(), 16, "a 128-byte block, doubleword by doubleword"); + assert_eq!(w[0], (PA & !127, 0), "the line at the tag's address"); + assert_eq!(w[2], (PA + 16, 0x2222_2222_1111_1111), "in memory order"); + } + } + } + + /// A cached store to a staged line stays in the cache: memory is not + /// written, and a later load sees it. A store elsewhere is memory's. + #[test] + fn a_store_to_a_staged_line_stays_in_the_cache() { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0); + assert_eq!(c.write::<4>(operand(PA, 0) + 4, PA + 4, 0xdead_beef), BUS_OK); + assert_eq!(c.write::<1>(operand(PA, 0) + 1, PA + 1, 0x7f), BUS_OK); + assert_eq!(load::<4>(&c, PA + 4), 0xdead_beef); + assert_eq!(load::<8>(&c, PA), 0x007f_0000_dead_beef); + assert!(bus.writes.lock().unwrap().is_empty()); + c.write::<4>(operand(PA, 0) + 0x1000, PA + 0x1000, 5); + assert_eq!(bus.writes.lock().unwrap().len(), 1); + } + + /// **Inferred** (NEC U10278EJ4V0UM p.246: a store to a CleanExclusive or + /// Shared block requests DirtyExclusive). A store moves the line to + /// DirtyExclusive, so invalidating with writeback sends the stored bytes + /// to memory instead of dropping them. Hit Invalidate still drops them. + #[test] + fn a_store_makes_the_line_dirty_and_the_writeback_carries_it() { + for state in [1, 2, 3] { + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, state); + stage_data(&c, PA, 0, 0); + c.write::<4>(operand(PA, 0), PA, 0x0123_4567); + assert_eq!((c.cache_op(C_ILT | CACH_SD, operand(PA, 0), 0) >> L2_STATE_SHIFT) & 3, 3, "state {state}"); + c.cache_op(C_IWBINV | CACH_SD, operand(PA, 0), 0); + let w = bus.writes.lock().unwrap().clone(); + assert_eq!(w.len(), 16, "state {state}"); + assert_eq!(w[0], (PA & !127, 0x0123_4567_0000_0000), "state {state}: the stored word, at the lower address"); + } + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0); + c.write::<4>(operand(PA, 0), PA, 1); + c.cache_op(C_HINV | CACH_SD, operand(PA, 0), PA); + assert!(bus.writes.lock().unwrap().is_empty(), "Hit Invalidate does not write back"); + // A load does not dirty a line. + let (c, bus) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 5); + load::<4>(&c, PA); + c.cache_op(C_IWBINV | CACH_SD, operand(PA, 0), 0); + assert!(bus.writes.lock().unwrap().is_empty()); + } + + /// op[1:0] = 10 is undefined on the R10000: it neither stages nor ends a line. + #[test] + fn the_undefined_secondary_instruction_encoding_does_nothing() { + let (c, bus) = staged_cache(); + c.cache_op(C_IST | CACH_SI, operand(PA & !127, 0), l2_tag_of(PA) | 2 << L2_STATE_SHIFT); + assert!(!c.page_staged(PA)); + assert_eq!(load::<8>(&c, PA), MEM); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 7); + c.cache_op(C_IWBINV | CACH_SI, operand(PA, 0), 0); + c.cache_op(C_HINV | CACH_SI, operand(PA, 0), PA); + c.cache_op(C_HWBINV | CACH_SI, operand(PA, 0), PA); + assert_ne!(load::<8>(&c, PA), MEM, "still staged"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 1); + } + + /// Datasheet step 6: the load's VA[13:12] may differ from TagLo[8:7]. + /// The lookup is physical, so the line hits whatever virtual address is used. + #[test] + fn a_different_virtual_index_still_hits() { + let (c, bus) = staged_cache(); + c.cache_op(C_IST | CACH_SD, operand(PA & !127, 0), l2_tag_of(PA) | 2 << L2_STATE_SHIFT | 3 << 7); + stage_data(&c, PA, 0, 0x1111_2222_3333_4444); + for va_bits in [0u64, 0x1000, 0x2000, 0x3000] { + let r = c.read::<8>(0xffff_ffff_8000_0000 | (PA & 0xfff) | va_bits, PA); + assert_eq!(r.data, 0x3333_4444_1111_2222, "VA[13:12] = {}", va_bits >> 12); + } + assert_eq!(bus.reads.load(Ordering::Relaxed), 0); + } + + /// The same rules at every geometry the family ships: 2 MiB with 128-byte + /// lines (Octane), 1 MiB with 64-byte lines (O2). + /// Two lines of one set differ by a way's worth of address. + #[test] + fn staging_works_at_every_cache_geometry() { + use crate::cpu::mips_cache_v2::MipsCache as _; + fn check>>(name: &str) { + let bus = Arc::new(Bus::default()); + let c = C::from(bus.clone() as Arc); + let line = C::L2_LINE as u64; + let way_bytes = (C::L2_SIZE / 2) as u64; + let (a, b) = (0x0020_3000u64 + line, 0x0020_3000u64 + line + way_bytes); + let op = |pa: u64, way: u64| 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8) | way; + let tag = |pa: u64, way: u64, state: u64| { + c.cache_op(C_IST | CACH_SD, op(pa & !(line - 1), way), l2_tag_of(pa) | state << L2_STATE_SHIFT); + }; + let data = |pa: u64, way: u64, d: u64| { c.cache_op(C_R10K_ISD | CACH_SD, op(pa & !7, way), d); }; + let load = |pa: u64| c.read::<8>(op(pa, 0), pa).data; + tag(a, 0, 2); data(a, 0, 0x0a0a_0a0a_0b0b_0b0b); + tag(b, 1, 2); data(b, 1, 0x0c0c_0c0c_0d0d_0d0d); + assert_eq!(load(a), 0x0b0b_0b0b_0a0a_0a0a, "{name}: way 0"); + assert_eq!(load(b), 0x0d0d_0d0d_0c0c_0c0c, "{name}: way 1"); + assert_eq!(load(a + 8), 0, "{name}: the line's unstaged doubleword"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "{name}"); + // Flip each tag address bit above the index. The first one is + // the other line's, which is staged in way 1. + assert_eq!(load(b), 0x0d0d_0d0d_0c0c_0c0c, "{name}: the other way's line"); + let index_top = (C::L2_SIZE / 2).trailing_zeros(); + for bit in (index_top + 1)..36 { + assert_eq!(load(a ^ (1 << bit)), MEM, "{name}: PA bit {bit}"); + } + // Dirty write back covers the whole line, at this cache's line size. + tag(a, 0, 3); + c.cache_op(C_IWBINV | CACH_SD, op(a, 0), 0); + let w = bus.writes.lock().unwrap().clone(); + assert_eq!(w.len() as u64, line / 8, "{name}"); + assert_eq!(w[0].0, a & !(line - 1), "{name}"); + assert_eq!(load(a), MEM, "{name}: ended"); + assert_ne!(load(b), MEM, "{name}: the other way stays"); + } + check::("R10000 1 MiB/128"); + check::("R10000 O2 1 MiB/64"); + check::("R12000 2 MiB/128"); + } + + /// The set of a secondary line is PA[N-2:B]. The geometry test's lines all + /// have the top index bit clear, which a lookup fixed at 4096 sets also + /// passes. Set the top bit (and the one below it) at every cache size and + /// find, load, hit-invalidate and index-invalidate the line. + #[test] + fn the_top_secondary_index_bit_reaches_the_line_at_every_cache_size() { + fn check>>(name: &str) { + let line = C::L2_LINE as u64; + let way_bytes = (C::L2_SIZE / 2) as u64; + let top = way_bytes >> 1; + for a in [0x0020_3000u64 + top, 0x0020_3000 + (top >> 1), 0x0020_3000 + top + (top >> 1)] { + let bus = Arc::new(Bus::default()); + let c = C::from(bus.clone() as Arc); + let op = |pa: u64, way: u64| 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8) | way; + let tag = |pa: u64, way: u64, state: u64| { + c.cache_op(C_IST | CACH_SD, op(pa & !(line - 1), way), l2_tag_of(pa) | state << L2_STATE_SHIFT); + }; + let load = |pa: u64| c.read::<8>(op(pa, 0), pa).data; + tag(a, 1, 2); + c.cache_op(C_R10K_ISD | CACH_SD, op(a & !7, 1), 0x0a0a_0a0a_0b0b_0b0b); + assert_eq!(load(a), 0x0b0b_0b0b_0a0a_0a0a, "{name}: {a:#x} hits"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "{name}: {a:#x}"); + // The line of the same set without the top bit is another set. + assert_eq!(load(a ^ top), MEM, "{name}: {a:#x} with the top bit flipped"); + c.cache_op(C_HINV | CACH_SD, op(a, 0), a); + assert_eq!(load(a), MEM, "{name}: {a:#x} Hit Invalidate ends it"); + tag(a, 1, 2); + c.cache_op(C_IWBINV | CACH_SD, op(a, 1), 0); + assert_eq!(load(a), MEM, "{name}: {a:#x} Index Writeback Invalidate ends it"); + } + } + check::("R10000 1 MiB/128"); + check::("R10000 O2 1 MiB/64"); + check::("R12000 2 MiB/128"); + } + + /// A snapshot carries the staged lines, and a restore ends any that were + /// staged before it. Dirty lines also reach memory on `writeback_all`. + #[test] + fn staged_lines_survive_a_snapshot_round_trip() { + let (c, _) = staged_cache(); + assert!(c.save_cache_state().as_table().unwrap().is_empty(), "nothing staged, nothing saved"); + stage_tag(&c, PA, 1, 3); + stage_data(&c, PA + 8, 1, 0x8000_0000_0000_0001); + c.cache_op(C_R10K_ISD | CACH_SD, operand(PA + 8, 1), 0x8000_0000_0000_0001); + let saved = c.save_cache_state(); + + let (d, bus) = staged_cache(); + stage_tag(&d, PA + 0x4000, 0, 2); // staged before the restore: must go + d.load_cache_state(&saved).unwrap(); + assert_eq!(load::<8>(&d, PA + 8), 0x0000_0001_8000_0000); + assert_eq!(load::<8>(&d, PA + 0x4000), MEM); + assert!(d.page_staged(PA) && !d.page_staged(PA + 0x4000)); + assert_eq!(bus.reads.load(Ordering::Relaxed), 1); + assert_eq!(d.cache_op(C_ILT | CACH_SD, operand(PA, 1), 0), l2_tag_of(PA) | 3 << L2_STATE_SHIFT); + + d.writeback_all(); + let w = bus.writes.lock().unwrap().clone(); + assert_eq!(w.len(), 16); + assert_eq!(w[1], (PA & !127 | 8, 0x0000_0001_8000_0000)); + assert_ne!(load::<8>(&d, PA + 8), MEM, "writeback_all keeps the line"); + + let empty = toml::Value::Table(Default::default()); + d.load_cache_state(&empty).unwrap(); + assert_eq!(load::<8>(&d, PA + 8), MEM, "restoring a snapshot with no staged lines ends them"); + } + + /// Datasheet check 7 (the R10000 cache index operations datasheet, "Mapped + /// operands and staging granularity"). Index Store Data (S) writes the one + /// quadword PA[6:4] names and leaves + /// the line's other quadwords alone, and a load or fetch finds each + /// quadword by the same bits. A quadword is stored once, the others are + /// rewritten many times, and the first must still read back what was + /// stored: for every quadword of the line, on 64- and 128-byte lines. + #[test] + fn index_store_data_writes_only_the_quadword_the_address_names() { + fn check>>(name: &str) { + let bus = Arc::new(Bus::default()); + let c = C::from(bus.clone() as Arc); + let line = C::L2_LINE as u64; + let pa0 = 0x0020_3000u64 & !(line - 1); + let op = |pa: u64| 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8); + let quads = line / 16; + for target in 0..quads { + c.cache_op(C_IST | CACH_SD, op(pa0), l2_tag_of(pa0) | 2 << L2_STATE_SHIFT); + // Clear the whole line, then store the target quadword once. + for q in 0..quads { + c.cache_op(C_R10K_ISD | CACH_SD, op(pa0 + 16 * q), 0); + } + // The upper doubleword of the quadword, to catch a mix-up of PA[3]. + let at = pa0 + 16 * target + 8; + let (w0, w1) = (0x1000_0000 + target as u32, 0x2000_0000 + target as u32); + c.cache_op(C_R10K_ISD | CACH_SD, op(at), (w1 as u64) << 32 | w0 as u64); + // Rewrite every other quadword many times, both halves. + for round in 0..40u64 { + for q in (0..quads).filter(|&q| q != target) { + let half = (round + q) & 1; + c.cache_op(C_R10K_ISD | CACH_SD, op(pa0 + 16 * q + 8 * half), 0xdead_0000_0000_0000 | round << 8 | q); + } + } + // Index Load Data (S) of the quadword returns what was stored. The + // check bits on the rewrites are assumed valid: none are modelled. + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, op(at), 0), (w1 as u64) << 32 | w0 as u64, + "{name} quadword {target}: Index Load Data"); + let rd = |pa: u64| c.read::<4>(op(pa), pa).data; + assert_eq!(rd(at), w0 as u64, "{name} quadword {target}: word at the lower address"); + assert_eq!(rd(at + 4), w1 as u64, "{name} quadword {target}: word at +4"); + assert_eq!(rd(at - 8), 0, "{name} quadword {target}: its other doubleword"); + assert_eq!(rd(at - 4), 0, "{name} quadword {target}: its other doubleword"); + let f = c.fetch(op(at), at); + assert_eq!(unsafe { (*f.instr).raw } as u64, w0 as u64, "{name} quadword {target}: fetch"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "{name}"); + // And the neighbours still hold what they were last given. + for q in (0..quads).filter(|&q| q != target) { + let half = (39 + q) & 1; + let v = 0xdead_0000_0000_0000u64 | 39 << 8 | q; + assert_eq!(c.read::<8>(op(pa0 + 16 * q + 8 * half), pa0 + 16 * q + 8 * half).data, v.rotate_left(32), + "{name} quadword {q}"); + } + } + } + check::("128-byte lines"); + check::("64-byte lines"); + } + + // ---- Mapped operands ------------------------------------------------ + + type MappedExec = crate::cpu::mips_exec::MipsExecutor; + + /// An executor, a cached 4 KiB page at 0x200000 mapped to frame `pfn` + /// with cache algorithm `alg` and dirty bit `dirty`, and the bus it sits on. + fn mapped(alg: u64, dirty: u64, pfn: u64) -> (Box, Arc) { + use crate::cpu::mips_exec::{MipsCpuConfig, MipsExecutor, EXEC_COMPLETE}; + let bus = Arc::new(Bus::default()); + let mut exec = Box::new(MipsExecutor::::new( + bus.clone() as Arc, crate::cpu::mips_tlb::MipsTlb::default(), + &MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))))); + #[cfg(feature = "jitv2")] + { exec.jitv2_dispatch_enabled = false; } + exec.core.cp0_config = (exec.core.cp0_config & !7) | 3; // KSEG0 cacheable + exec.core.cp0_status &= !crate::cpu::mips_core::STATUS_ERL; + exec.resync_privilege_state(); + exec.core.cp0_index = 3; + exec.core.cp0_pagemask = 0; + exec.core.cp0_entryhi = 0x100 << 13; + exec.core.cp0_entrylo0 = (pfn << 6) | (alg << 3) | (dirty << 2) | (1 << 1); + exec.core.cp0_entrylo1 = 0; + use crate::cpu::mips_isa::OP_COP0; + assert_eq!(exec.exec((OP_COP0 << 26) | (0x10 << 21) | 0x02), EXEC_COMPLETE); // TLBWI + (exec, bus) + } + + /// CACHE `op` on `va`, with TagLo/TagHi as given; the instruction's status. + fn cache_at(exec: &mut MappedExec, op: u32, va: u64, tag: u64) -> u32 { + exec.core.gpr[4] = va; + exec.core.cp0_taglo = tag & 0xffff_ffff; + exec.core.cp0_taghi = (tag >> 32) as u32; + exec.exec((crate::cpu::mips_isa::OP_CACHE << 26) | (4 << 21) | (op << 16)) + } + + const VA: u64 = 0x20_0000; + + /// The executor is large: run on a big stack, as the other TLB tests do. + fn big_stack(f: impl FnOnce() + Send + 'static) { + std::thread::Builder::new().stack_size(16 * 1024 * 1024).spawn(f).unwrap().join().unwrap(); + } + + /// The secondary set and way come from the physical address of a mapped + /// operand (NEC U12754EJ1V0UMJ1 pp.14-15; datasheet "Mapped operands", + /// pending its review). Stage through the mapped page, then a cached + /// access to the frame's kseg0 address hits, and the tag is at the frame's + /// set, not the virtual address's. + #[test] + fn a_mapped_index_operation_lands_on_the_physical_set() { + big_stack(|| { + use crate::cpu::mips_exec::EXEC_COMPLETE; + let pfn = 0x3a5; // PA 0x3a5000: set 0x4a0; the VA's set is 0 + let (mut exec, bus) = mapped(3, 1, pfn); + let pa = pfn << 12 | 0x40; + let tag = l2_tag_of(pa) | 2 << L2_STATE_SHIFT; + assert_eq!(cache_at(&mut exec, 0x0b, VA + 0x40, tag), EXEC_COMPLETE); + assert_eq!(cache_at(&mut exec, 0x1f, VA + 0x40, 0x3333_4444_1111_2222), EXEC_COMPLETE); + let kseg0 = 0xffff_ffff_8000_0000 | pa; + assert_eq!(exec.cache.read::<4>(kseg0, pa).data, 0x1111_2222); + assert_eq!(exec.cache.read::<4>(kseg0 + 4, pa + 4).data, 0x3333_4444); + let f = exec.cache.fetch(kseg0, pa); + assert_eq!(unsafe { (*f.instr).raw }, 0x1111_2222, "a cached fetch through the frame"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0); + // The tag is at the physical set, read back through an unmapped operand. + assert_eq!(cache_at(&mut exec, 0x07, 0xffff_ffff_8000_0000 | (pa & !127), 0), EXEC_COMPLETE); + assert_eq!(exec.core.cp0_taglo | (exec.core.cp0_taghi as u64) << 32, tag); + // And nothing is at the set the virtual address would have chosen. + assert_eq!(cache_at(&mut exec, 0x07, 0xffff_ffff_8000_0000 | VA, 0), EXEC_COMPLETE); + assert_eq!(exec.core.cp0_taglo, 0); + // Index Writeback Invalidate through the mapped operand ends the line. + assert_eq!(cache_at(&mut exec, 0x03, VA + 0x40, 0), EXEC_COMPLETE); + assert_eq!(exec.cache.read::<4>(kseg0, pa).data, 0xeeee_eeee); + }) + + } + + /// Two virtual pages on one frame reach one secondary line; the primary + /// index stays virtual (datasheet checks 2 and 3). + #[test] + fn aliases_share_a_secondary_line_and_the_primary_stays_virtual() { + big_stack(|| { + use crate::cpu::mips_exec::EXEC_COMPLETE; + let (mut exec, _) = mapped(3, 1, 0x3a5); + let pa = 0x3a5000u64; + let tag = l2_tag_of(pa) | 2 << L2_STATE_SHIFT; + cache_at(&mut exec, 0x0b, VA, tag); + // Another way to the same line: the unmapped address of the frame. + assert_eq!(cache_at(&mut exec, 0x07, 0xffff_ffff_8000_0000 | pa, 0), EXEC_COMPLETE); + assert_eq!(exec.core.cp0_taglo | (exec.core.cp0_taghi as u64) << 32, tag); + // Primary D: indexed by VA[13:5]. Mapped store, unmapped load at the + // same low 14 bits but another frame. + assert_eq!(cache_at(&mut exec, 0x09, VA + 0x40, 0x0000_0abc), EXEC_COMPLETE); + assert_eq!(cache_at(&mut exec, 0x05, 0xffff_ffff_8000_0000 | 0x40, 0), EXEC_COMPLETE); + assert_eq!(exec.core.cp0_taglo, 0x0abc); + }) + + } + + /// A mapped operand takes TLB Refill when no entry maps it, and the cache + /// does not change. + #[test] + fn an_unmapped_operand_takes_tlb_refill() { + big_stack(|| { + use crate::cpu::mips_exec::{EXEC_IS_EXCEPTION, EXEC_IS_TLB_REFILL}; + let (mut exec, _) = mapped(3, 1, 0x3a5); + let st = cache_at(&mut exec, 0x0b, 0x60_0000, l2_tag_of(0x3a5000) | 2 << L2_STATE_SHIFT); + assert!(st & EXEC_IS_EXCEPTION != 0 && st & EXEC_IS_TLB_REFILL != 0, "status {st:#x}"); + assert!(!exec.cache.page_staged(0x3a5000)); + // The same through an index operation on a data cache. + let st = cache_at(&mut exec, 0x1d, 0x60_0000, 0); + assert!(st & EXEC_IS_EXCEPTION != 0 && st & EXEC_IS_TLB_REFILL != 0, "status {st:#x}"); + }) + + } + + /// A valid entry with the dirty bit clear is accepted: CACHE never takes + /// TLB Modified. + #[test] + fn a_clean_page_takes_no_tlb_modified() { + big_stack(|| { + use crate::cpu::mips_exec::EXEC_COMPLETE; + let (mut exec, _) = mapped(3, 0, 0x3a5); + let tag = l2_tag_of(0x3a5000) | 2 << L2_STATE_SHIFT; + assert_eq!(cache_at(&mut exec, 0x0b, VA, tag), EXEC_COMPLETE); + assert_eq!(cache_at(&mut exec, 0x1f, VA, 7), EXEC_COMPLETE); + assert_eq!(cache_at(&mut exec, 0x03, VA, 0), EXEC_COMPLETE); + }) + + } + + /// A CACHE operand on an uncached page is undefined on the R10000. This + /// model translates it and goes ahead as if the page were cached; pinned + /// here so a change is deliberate. + #[test] + fn an_operand_on_an_uncached_page_goes_ahead() { + big_stack(|| { + use crate::cpu::mips_exec::EXEC_COMPLETE; + let (mut exec, _) = mapped(2, 1, 0x3a5); + let tag = l2_tag_of(0x3a5000) | 2 << L2_STATE_SHIFT; + assert_eq!(cache_at(&mut exec, 0x0b, VA, tag), EXEC_COMPLETE); + assert!(exec.cache.page_staged(0x3a5000)); + }) + + } + + /// The cache holds staged lines from the first valid line to the end of + /// the last, and reports each of those two changes once. + #[test] + fn the_cache_holds_staged_lines_from_the_first_to_the_last() { + let (c, _) = staged_cache(); + assert!(!c.holds_staged_lines() && !c.take_staged_changed()); + stage_tag(&c, PA, 0, 2); + assert!(c.holds_staged_lines() && c.take_staged_changed()); + assert!(!c.take_staged_changed(), "reading clears it"); + stage_tag(&c, PA + 0x1000, 0, 1); // a second line: no change + stage_tag(&c, PA, 0, 0); + assert!(c.holds_staged_lines() && !c.take_staged_changed()); + stage_tag(&c, PA + 0x1000, 0, 0); + assert!(!c.holds_staged_lines() && c.take_staged_changed()); + } + + /// While lines are staged, a cached access to RAM the window maps goes + /// through the bus path, where the staged line is found: a load of it + /// hits, and a load elsewhere reads the bus, not the window. With the + /// last line gone the window serves RAM again. + #[test] + fn staged_lines_take_ram_off_the_window_until_the_last_one_ends() { + let (c, bus) = staged_cache(); + let mut window = vec![0x5555_5555u32; 0x4000]; // 64 KiB of RAM + let pa = 0x2000u64; + unsafe { c.set_tcache_window(window.as_mut_ptr() as *mut u8) }; + unsafe { *c.tcache_bitmap_ptr() = 1 }; + assert_eq!(load::<4>(&c, pa), 0x5555_5555, "the window"); + stage_tag(&c, pa, 0, 2); + stage_data(&c, pa, 0, 0x1111_2222_3333_4444); + assert_eq!(load::<4>(&c, pa), 0x3333_4444, "the staged line"); + assert_eq!(load::<4>(&c, pa + 0x100), MEM & 0xffff_ffff, "the bus, not the window"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 1); + stage_tag(&c, pa, 0, 0); + assert_eq!(load::<4>(&c, pa + 0x100), 0x5555_5555, "the window again"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 1); + } + + /// Only the R10000 family stages lines: the R4400's index operations are + /// as before. + #[test] + fn the_r4400_does_not_stage_lines() { + let c = R4400ShadowCache::from(Arc::new(Bus::default()) as Arc); + c.cache_op(C_IST | CACH_SD, 0x8010_1000, 0x0000_0800 | 4 << 14); + assert!(!c.page_staged(PA)); + assert_eq!(c.read::<8>(0x8010_1000, PA).data, MEM); + } + /// Under tcache the JIT serves this model through the window alone, and /// only once both windows are published: the emitted code dereferences /// them without a null check. diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 2b92dc849..5b7ef9130 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -104,10 +104,9 @@ impl CacheOp { let op = cache_op & 0x1C; let sel = cache_op & 3; // The R10000 index operations that name a cache entry rather than an - // address: they are not translated. The primary Index Load/Store Data - // operations select by virtual address, but a mapped one still takes - // TLB Refill and Invalid (VR10000 Instruction Set UM p. 15), so they - // are. + // address. They still translate their operand (see `translates`), but + // the primary Index Load/Store Data operations select by virtual + // address. let primary_data = r10k && (op == C_R10K_ILD || op == C_R10K_ISD) && (sel == CACH_PI || sel == CACH_PD); @@ -117,7 +116,11 @@ impl CacheOp { || (op == C_R10K_ISD && (sel == CACH_SI || sel == CACH_SD))); let hit_op = op == C_CDX || op == C_HINV || op == C_HWBINV || op == C_HWB || op == C_HSV; CacheOp { - translates: primary_data || (!index_op && hit_op), + // The R10000 translates every operand but Cache Barrier's, a + // mapped one taking TLB Refill and Invalid (VR10000 Instruction + // Set UM p. 15; the R10000 cache index operations datasheet, "Mapped + // operands"). Older models translate the hit operations only. + translates: if r10k { !(op == C_R10K_CBARRIER && sel == CACH_PI) } else { hit_op }, stores_tag: op == C_IST || ((index_op || primary_data) && op == C_R10K_ISD), loads_tag: op == C_ILT || (index_op && op == C_R10K_ILD), // Index_Load_Tag also loads CP0 ECC with the selected doubleword's @@ -693,6 +696,19 @@ pub trait MipsCache: Send + Sync { /// Default `false`: a model that keeps instruction-cache state does. fn fetch_is_pure(&self) -> bool { false } + /// Does the cache hold a line the guest staged with Index Store Tag and + /// Index Store Data, which cached accesses must be served from instead of + /// memory? While it does, the cache serves every cached access itself, + /// and the executor runs no compiled code: the JIT's inline loads and + /// stores reach RAM without calling the cache, and it compiled from + /// memory. Default `false`: a model that serves every access itself has + /// no such line. + fn holds_staged_lines(&self) -> bool { false } + + /// Did `holds_staged_lines` change since the last call? Reading it clears + /// it. Asked after each CACHE instruction on the R10000 family. + fn take_staged_changed(&self) -> bool { false } + /// Write every dirty line back to memory, so RAM holds everything. /// Default no-op: a cache that holds no data has nothing to write. fn writeback_all(&self) {} diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 7a5edf710..5977dcb30 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -3359,6 +3359,28 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.sync_cache_error_control(); } + /// The cache began or stopped holding lines the guest staged + /// (`MipsCache::holds_staged_lines`). While it holds any it serves every + /// cached access itself, so nothing may reach RAM around it: the JIT's + /// inline loads and stores go through the micro-TLB and ppmem's window, + /// and its code was compiled from memory. So no code page is tracked + /// while lines are held, which runs everything in the interpreter, and the + /// translations cached before the change go. Only the shipped PROMs stage + /// lines, and they end them before the kernel runs. + #[cold] + #[inline(never)] + pub(crate) fn staged_lines_changed(&mut self) { + self.nanotlb_invalidate(); + #[cfg(feature = "jitv2")] + { + self.jitv2_frames = if self.cache.holds_staged_lines() { + 0 + } else { + crate::cpu::jitv2::jitv2::frames_for(&self.memory) + }; + } + } + /// Tell the cache which Status bits govern parity: CE forces the ECC /// register into stored check bits, and DE or ERL turn detection off. /// Every Status change comes through `resync_privilege_state`. @@ -6647,23 +6669,27 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let cache_op = d.rt as u32; // Encoded operation: cache_sel in bits[1:0], op in bits[4:2] let op = cache_op & 0x1C; - // Hit operations need address translation; index operations use the - // address as an index. On an R10000 the encodings for 5/6/7 are index - // operations, not the R4000 hit operations: translating them would - // fault on an index that is not a valid virtual address (see - // C_R10K_ISD in mips_cache_v2). `CpuModel::CACHE_OPS` holds that - // decision per encoding, so the choice here is one bit test. + // On the R10000 every CACHE operand but Cache Barrier is translated + // like a data address before the operation, index operations included + // (NEC U12754EJ1V0UMJ1 pp.70, 72, the R10000 cache index operations datasheet, + // "Mapped operands"). A mapped operand can take TLB Refill, TLB + // Invalid or a privilege Address Error. It is translated as a read, so + // it never takes TLB Modified. The primary caches index by the virtual + // address and only use the translation for its exceptions. The + // secondary cache takes its set and way from the physical address, + // passed in place of the virtual one. An operand on an uncached page is undefined on + // the R10000; it is translated and the operation goes ahead as if the + // page were cached. `CpuModel::CACHE_OPS` holds the decision per + // encoding, so the choice here is one bit test. let sel = cache_op & 3; let bit = 1u32 << cache_op; let needs_translation = C::CACHE_OPS.translates & bit != 0; let phys_addr = if needs_translation { - // Hit operations need address translation let tr = self.nanotlb_translate::<{AccessType::Read as u8}>(virt_addr); if tr.is_exception() { return self.handle_exception(tr.status); } tr.phys } else { - // Index operations use virt_addr as index, no translation needed virt_addr }; @@ -6723,7 +6749,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } // Call unified cache interface - let result = self.cache.cache_op(cache_op, virt_addr, phys_addr_or_taglo); + let index_addr = if C::R10K_CACHE_OPS && sel == CACH_SD { phys_addr } else { virt_addr }; + let result = self.cache.cache_op(cache_op, index_addr, phys_addr_or_taglo); + if C::R10K_CACHE_OPS && self.cache.take_staged_changed() { + self.staged_lines_changed(); + } // An I-cache write-back of a line with bad parity raises a Cache // Error from the CACHE instruction itself. #[cfg(feature = "cache-ecc")] @@ -11100,6 +11130,9 @@ impl Resettable for MipsC exec.core.reset(false); exec.tlb.power_on(); exec.cache.power_on(); + if exec.cache.take_staged_changed() { + exec.staged_lines_changed(); + } exec.core.in_delay_slot = false; exec.core.delay_slot_target = 0; #[cfg(feature = "developer")] @@ -11243,6 +11276,9 @@ impl Saveable for MipsCpu if let Some(cache_v) = get_field(v, "cache") { exec.cache.load_cache_state(cache_v)?; + if exec.cache.take_staged_changed() { + exec.staged_lines_changed(); + } } // cp0_status was restored above by direct field write, so translate_fn and the From 4eba0c8800d25e51d53a5caa3265a4c17798a07a Mon Sep 17 00:00:00 2001 From: James Braid Date: Sat, 7 Nov 2026 07:20:00 -0800 Subject: [PATCH 7/7] cpu: serve fetches from staged primary instruction-cache lines A PROM can also stage a line in the primary instruction cache with Index Store Tag (PI) and Index Store Data (PI) and run from it. A fetch whose primary tag matches now executes the staged word, ahead of a staged secondary line, and the lines end on Index Invalidate, Hit Invalidate, a tag stored invalid, or power-on. A primary word is 36 bits of predecoded instruction plus parity. Fetch decodes the predecode classes the manual describes (eleven, each row checked against the encoding table) and checks even parity over the 36 bits and ECC[0]. Status.DE and ERL suppress the Cache Error. A word of a class the model does not know is refused: it is reported once per class and the fetch reads the staged secondary line, else memory, instead. That is an approximation, since hardware would run the word. A fetch that misses every primary way and hits a staged secondary line refills a 64-byte primary block from it, into the invalid way first. The refilled words keep a flag beside the parity so that Index Load Data followed by Index Store Data copies them verbatim; a guest-built word is never taken for one. The set of words handed out is bounded and not saved in snapshots. Invalidating a secondary line purges the primary instruction and data lines of its block that the tag's virtual index (TagLo[8:7]) names, with the D-side writeback the operation implies. A valid primary line holds the cache off its window paths as a secondary one does. The primary index is virtual. While lines are held the decode cache reads every word through a StagedBus of its own, which takes the virtual address from the PC: every fetch is at the PC, and a fused pair never leaves its I-cache line. The parity check reads DE and ERL from Status when it runs. The executor hands the cache both registers when it starts holding lines, so neither costs anything as the CPU runs. --- clanker/cpu/staged-secondary-cache-lines.md | 77 +- src/cpu/jitv2/equiv_test.rs | 67 + src/cpu/mips_cache_shadow.rs | 1440 +++++++++++++++++-- src/cpu/mips_cache_v2.rs | 8 + src/cpu/mips_exec.rs | 3 + 5 files changed, 1508 insertions(+), 87 deletions(-) diff --git a/clanker/cpu/staged-secondary-cache-lines.md b/clanker/cpu/staged-secondary-cache-lines.md index acdacac84..2e3150d9a 100644 --- a/clanker/cpu/staged-secondary-cache-lines.md +++ b/clanker/cpu/staged-secondary-cache-lines.md @@ -1,9 +1,9 @@ -# Staged secondary-cache lines: a hit until an explicit operation ends them +# Staged cache lines: a hit until an explicit operation ends them The R10000-family shadow cache (`src/cpu/mips_cache_shadow.rs`) serves a cached load, store or fetch from a secondary line the guest staged with Index Store Tag (S) and Index Store Data (S): a valid state in TagLo[11:10] and a tag equal to -PA[39:18] of the access. The rules are the R10000 cache index operations datasheet's +PA[39:18] of the access. The rules are those of the R10000 cache index operations datasheet. A guest that never stages a line pays nothing: no access is tested for one. @@ -20,6 +20,66 @@ Staged lines left at the login prompt, `l2 probe 0` in the monitor, CPU 0: Octane (IP30 PROM, IRIX 6.5.30) 0, O2 R10000 (IRIX 6.5.30) 0. The shipped PROMs invalidate what they stage, so the machine is back on the window paths and the JIT before IRIX runs. +## Primary instruction-cache lines + +Index Store Tag (I) and Index Store Data (I) stage a primary I line the same +way: tag TagLo[31:8] = PA[35:12], TagLo[6] valid, TagHi[3:0] = PA[39:36]; index +VA[13:6], way VA[0], one word per Index Store Data at VA[13:2], 16 per block. A +cached fetch checks both ways at its VA[13:6] against its PA, before the +secondary cache. Index Invalidate (I), Hit Invalidate (I), an Index Store Tag +with the valid bit clear, and power-on end the line. Loads are not served by it. +Model choices: the compare on a fetch is inferred and a hit does not touch LRU. + +The stored word is the 36-bit predecoded instruction plus even parity, not an +instruction in TagLo. A fetch hit decodes it with the layout the R10000 cache index operations datasheet's +`r10000-cache-index-ops.md` infers from a shipped PROM's table, for sampled +opcodes only (`l1i_decode`, `L1I_CLASSES`). `addiu $4,$0,2` is `0x3_4420_0004`. +- A word of a class not in the table is refused: reported once per class on + stderr, and the fetch is treated as a miss and reads the staged secondary line + if there is one, else memory. An approximation, not + hardware behavior (hardware runs the word). Chosen over a Cache Error because + memory holds the instruction a word was built from in all but a few stale + entries, so a miss executes the right instruction for nearly every staged + word, where an error stops a guest the hardware runs. The limit: a staged + entry that is stale against memory runs the memory word, not the staged one, + which is why it is reported. +- The class table is the 11 classes of the datasheet (0x00 nop, 0x08 dsll/dsll32, + 0x18 or, 0x1a addi/addiu/ori/lui, 0x1e daddiu, 0x35 cache, 0x36 ld, 0x3f mtc0, + 0x40 jal, 0x41 jr/j, 0x71 REGIMM), all inferred. Class, p[28:26] and, for the + SPECIAL classes, the function field identify the instruction; a known class + with anything else is refused too. +- A cached fetch that misses the primary I cache and hits a staged secondary + line refills a 64-byte primary block from it (the only refill; ordinary + fetches are not tracked). The victim is an Invalid way before a Valid one, and + way 0 when both are valid: a model choice, the sheet states no victim rule. The + words are stored in the raw encoding with even parity, the architectural word + carried in place of hardware's predecoded form. The block is tagged with the + fetch's PA, valid, so Index Load Tag/Data (I) return it and a readback stored + verbatim into another line executes. +- Index Writeback Invalidate (S), Hit Invalidate (S) and Hit Writeback + Invalidate (S) purge the primary I and D lines inside the secondary block, + found through the stored tag's physical block and its virtual index + (TagLo[8:7]): a line of the block at another VA[13:12] survives. A line the + model refilled itself is purged whatever its index, because its index is the + fetch's and the model leaves the guest's staged tag alone. Approximation: the + hardware would have recorded the fetch's index in the tag. +- Parity is checked on a fetch hit: even over the 36 bits and ECC[0]. A bad word + posts a Cache Error (CacheErr.ED) unless Status.DE or ERL is set. +- Index Load Data (I) returns the stored word and parity as stored, so a readback + stored back verbatim round-trips and executes. A line the guest never staged + and the model did not refill reads back as the shadow's contents, not as the + predecoded memory word. A word with p[35:32] = 0xf and the model's raw flag + (set by a refill, kept by a verbatim Index Store Data of a word Index Load + Data handed out) is this model's own encoding of a raw instruction in p[31:0] + (`L1I_RAW`), so a word outside the sampled set can round-trip. Model choice; + the hardware contents are unknown. + +The primary data cache is not served: loads never hit a line staged in it. Its +tags and words are the diagnostic shadow, and only the inclusion purge acts on +them: a secondary invalidate clears the state and modifier of its lines, and +the writeback forms first copy a line with modifier 010 into the secondary +line, which a DirtyExclusive block then sends to memory. + ## How staged lines reach the data path No access is checked against them. While the cache holds any, it takes RAM @@ -30,7 +90,9 @@ off its own fast paths instead (`serve_window`): through a `StagedBus`: staged lines answer for their addresses, memory for the rest; - the decode cache gets no window (`fetch_window`), so it reads every word - through the same route; + through a `StagedBus` too. That one also finds staged primary instruction + lines, which are indexed by virtual address: every fetch is at the PC, which + the executor hands the cache with Status (`set_cpu_registers`); - the executor runs no compiled code (`staged_lines_changed`). The JIT's inline loads and stores reach RAM through the micro-TLB and ppmem's window without calling the cache, and it compiles from memory. The executor learns of the @@ -54,9 +116,14 @@ Dropping the executor's part fails `a_compiled_load_hits_a_staged_line_not_the_w - Every CACHE operand is translated first (exceptions: TLB Refill/Invalid, privilege Address Error, never TLB Modified). The secondary set, way and quadword come from the operand's physical address, which the executor passes in place of the virtual one, the primaries' from its virtual address. An operand on an uncached page is undefined on the R10000 and goes ahead as if cached. - The datasheet clears the R10000. The R12000 and R14000 use the same tag layout here, which is an inference. -- Not modelled: predecode, primary-cache staged hits, ECC and check bits, the - virtual-index purge, the BadVAddr/Context/EntryHi values after a CACHE-raised TLB exception, Status.CH, the +- Not modelled: the predecode of unsampled classes, primary data cache staged + hits, ECC and check bits, the purge a cached access makes when its VA[13:12] + differs from the tag's recorded index, the BadVAddr/Context/EntryHi values after a CACHE-raised TLB exception, Status.CH, the tag rewrite on invalidate (only the state clears), coherence with other CPUs and DMA. - Staged lines are saved in snapshots (`save_cache_state`). The rest of the shadow arrays, which only diagnostics read, are not. +- The raw words Index Load Data handed out (`l1i_issued`) are bounded at 64 Ki + and not saved. A raw word read before a snapshot and stored after it is stored + without the model's flag, so it is refused and reported like a guest-built + 0xf word; it never runs as raw bits. diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index ee4a0d651..57c757084 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -4510,6 +4510,73 @@ mod tests { assert!(!exec.pcp.is_null(), "the page is back on the JIT"); } + /// The same for a line staged in the primary instruction cache with + /// Index Store Tag (I) and 16 Index Store Data (I), on way 1: compiled + /// code for the page is dropped, the staged words run, and an Index + /// Invalidate (I) puts the page back on memory and the JIT. + #[test] + fn a_staged_primary_instruction_line_is_fetched_instead_of_the_compiled_page() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + use crate::cpu::mips_isa::{OP_ADDIU, OP_CACHE}; + let pc = 0xFFFF_FFFF_8000_1000u64; + let ra = 0xFFFF_FFFF_8000_2000u64; + let addiu = |imm: u32| make_i(OP_ADDIU, 0, 2, imm as u16); + let jr = make_r(crate::cpu::mips_isa::OP_SPECIAL, 31, 0, 0, 0, crate::cpu::mips_isa::FUNCT_JR); + let mem = Arc::new(MockMemory::new()); + for (i, w) in [addiu(0x11), jr, 0].into_iter().enumerate() { + mem.set_word(0x1000 + 4 * i as u64, w); + } + let bus: Arc = mem.clone(); + let mut exec = Box::new(MipsExecutor::::new( + bus, PassthroughTlb::default(), &MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))))); + exec.core.cp0_config = (exec.core.cp0_config & !7) | 3; + exec.install_jit_hooks(); + exec.jitv2_inline_compile = true; + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + let run = |exec: &mut MipsExecutor| -> u64 { + exec.core.pc = pc; + exec.core.gpr[31] = ra; + exec.core.gpr[2] = 0; + for _ in 0..16 { + if exec.core.pc == ra { break; } + exec.step_jit(); + } + assert_eq!(exec.core.pc, ra, "the code returns"); + exec.core.gpr[2] + }; + let cache_op = |exec: &mut MipsExecutor, op: u32, off: u64, lo: u32, hi: u32| { + exec.core.gpr[4] = pc + off; + exec.core.cp0_taglo = lo as u64; + exec.core.cp0_taghi = hi; + assert_eq!(exec.exec(make_i(OP_CACHE, 4, op, 0)), crate::cpu::mips_exec::EXEC_COMPLETE); + }; + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x11); + } + assert!(unsafe { (*exec.pcp).is_published(0) }, "compiled"); + // Block at PA 0x1000: tag PA[35:12] = 1 in TagLo[31:8], valid in bit 6. + cache_op(&mut exec, 0x08, 1, (1 << 8) | (1 << 6), 0); + assert!(exec.pcp.is_null(), "staging dropped the page from the JIT"); + let words = [addiu(0x22), jr]; + for w in 0..16u64 { + let word = words.get(w as usize).copied().unwrap_or(0); + // The model's raw encoding (p[35:32] = 0xf), with even parity. + exec.cache.issue_raw_for_test(0xfu64 << 32 | word as u64); + exec.core.cp0_ecc = (0xfu64 << 32 | word as u64).count_ones() & 1; + cache_op(&mut exec, 0x1c, 4 * w + 1, word, 0xf); + } + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x22, "the staged words run, not the compiled page"); + } + cache_op(&mut exec, 0x00, 1, 0, 0); // Index Invalidate (I), way 1 + for _ in 0..4 { + assert_eq!(run(&mut exec), 0x11); + } + assert!(!exec.pcp.is_null(), "back on the JIT"); + } + /// The data side: compiled loads reach RAM through the micro-TLB and /// ppmem's window, never the cache. Warm the micro-TLB on a page, stage a /// secondary line in it, and the compiled load returns the staged word; diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 84e13dc25..5e9547f9c 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -31,7 +31,11 @@ //! that costs a normal access nothing: on the R10000 family, a //! secondary line the guest *staged* — a valid tag written by Index Store Tag //! and data by Index Store Data — is a real hit for later cached loads, stores -//! and fetches, served from the shadow without a bus access. See [`Staged`]. +//! and fetches, served from the shadow without a bus access. A line staged +//! in the primary instruction cache the same way serves fetches, before the +//! secondary cache. Its words are stored predecoded, and decode back for the +//! sampled classes only (`l1i_decode`); other classes read memory, with a +//! report. See [`Staged`]. //! Nothing else about the shadow reaches the data path: a line the guest never //! staged is memory, as before, and ordinary misses never fill or evict. So a //! staged line lives until an explicit index or hit operation, an Index Store @@ -43,8 +47,9 @@ use std::sync::Arc; use crate::cpu::mips_cache_v2::{ cache_op_name, model, r4k_load_taglo, r4k_store_taglo, CpuModel, FetchInstrResult, MipsCache, - C_HINV, C_HWBINV, C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, + C_HINV, C_HWBINV, C_IINV, C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, }; +use crate::cpu::mips_exec::EXEC_CACHE_ERR; use crate::traits::{BusDevice, BusRead64, BUS_OK}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_OP, devlog_is_active, devlog_mask}; @@ -110,6 +115,130 @@ const L2_TAG_PA_MASK: u64 = 0xf_ffff_c000; /// 0 Invalid, 1 Shared, 2 CleanExclusive, 3 DirtyExclusive. const L2_STATE_SHIFT: u32 = 10; const L2_STATE_DIRTY_EXCLUSIVE: u64 = 3; +/// The secondary tag's virtual index, TagLo[8:7]: VA[13:12] of the access that +/// filled the line, which names the primary sets its subset lines are in. +const L2_VINDEX_SHIFT: u32 = 7; + +/// The primary data tag's state (TagLo[7:6], 0 Invalid) and its state modifier +/// (TagHi[31:29], kept at bits 63:61 of the assembled tag): 010 is a line +/// written and inconsistent with the secondary cache. +const L1D_STATE_SHIFT: u32 = 6; +const L1D_MODIFIER_SHIFT: u32 = 61; +const L1D_MODIFIER_WRITTEN: u64 = 2; + +/// The primary instruction-cache tag, from the register pair (NEC +/// U10278EJ4V0UM pp.267-270; the R10000 cache index operations datasheet, "Tag +/// operations"): TagLo[31:8] is PA[35:12], TagLo[6] the state (1 valid), and +/// TagHi[3:0] PA[39:36], kept at bits 35:32 of the assembled tag. +const L1I_TAG_PA_MASK: u64 = 0xf_ffff_ff00; +const L1I_VALID: u64 = 1 << 6; + +/// Most raw words `l1i_issued` holds before it starts again. +const L1I_ISSUED_MAX: usize = 1 << 16; + +/// CacheErr.ED: a data-field error (NEC U10278EJ4V0UM p.265). +const L1I_CACHEERR_ED: u32 = 1 << 29; + +/// Even parity: the ones in the 36 data bits plus the parity bit total an even +/// number (NEC U10278EJ4V0UM pp.190-191). +#[inline(always)] +fn l1i_parity_ok(p: u64, parity: u32) -> bool { + (p.count_ones() + parity) & 1 == 0 +} + +/// The parity bit that makes `p` even. +#[cfg(test)] +fn l1i_parity_of(p: u64) -> u32 { + p.count_ones() & 1 +} + +/// A primary instruction-cache word is the instruction predecoded: 36 bits +/// that fetch decodes back. Neither NEC manual gives the layout. This is the +/// one in the R10000 cache index operations datasheet ("Primary instruction-cache data +/// word", point 5) infers from a shipped PROM's table, for sampled opcodes +/// only, with `p` the 36-bit word and `f` the instruction: +/// p[25:20] = f[5:0], p[19:15] = f[10:6], p[14:10] = f[25:21], +/// p[28:26] = f[28:26], p[35:29] = a 7-bit class, +/// p[4:0] and p[9:5] = f[20:16] and f[15:11] (rt and rd positions), in the +/// order the opcode gives. +/// f[31:29] is not in any field, so it comes from the class. The class table +/// is what the sheet samples, and it is all this model knows: an entry gives +/// the class, f[31:29], and whether p[4:0] is the rt position. Every other +/// class is refused (`Err(class)`), never guessed. +/// +/// Example from the sheet: `addiu $4,$0,2` (0x24040002) is 0x3_4420_0004. +const L1I_CLASSES: &[L1iClass] = &[ + // The table of the datasheet, every row inferred. + L1iClass { class: 0x00, hi3: 0b000, rt_low: false, ops: &[0], funct: &[0x00] }, + L1iClass { class: 0x08, hi3: 0b000, rt_low: false, ops: &[0], funct: &[0x38, 0x3c] }, // dsll, dsll32 + L1iClass { class: 0x18, hi3: 0b000, rt_low: false, ops: &[0], funct: &[0x25] }, // or + L1iClass { class: 0x1a, hi3: 0b001, rt_low: true, ops: &[0, 1, 5, 7], funct: &[] }, // addi addiu ori lui + L1iClass { class: 0x1e, hi3: 0b011, rt_low: true, ops: &[1], funct: &[] }, // daddiu + L1iClass { class: 0x35, hi3: 0b101, rt_low: false, ops: &[7], funct: &[] }, // cache + L1iClass { class: 0x36, hi3: 0b110, rt_low: true, ops: &[7], funct: &[] }, // ld + L1iClass { class: 0x3f, hi3: 0b010, rt_low: false, ops: &[0], funct: &[] }, // mtc0 + L1iClass { class: 0x40, hi3: 0b000, rt_low: false, ops: &[3], funct: &[] }, // jal + L1iClass { class: 0x41, hi3: 0b000, rt_low: false, ops: &[0, 2], funct: &[0x08] }, // jr (op 0, funct 8), j (op 2) + L1iClass { class: 0x71, hi3: 0b000, rt_low: false, ops: &[1], funct: &[] }, // REGIMM +]; + +/// One row of the class table. `ops` are the p[28:26] values (f[28:26]) the +/// class holds, `funct` the function fields allowed for its SPECIAL members +/// (checked when p[28:26] is 0 and the class is a SPECIAL one), `rt_low` +/// whether p[4:0] is the rt position (else the rd position). +struct L1iClass { + class: u8, + hi3: u32, + rt_low: bool, + ops: &'static [u8], + funct: &'static [u8], +} + +/// **Model choice, hardware contents unknown.** A word whose p[35:32] is 0xf +/// and that carries `L1I_RAW_FLAG` is this model's encoding of a raw +/// instruction: p[31:0] is the instruction. A refill writes it, Index Load +/// Data returns it and a verbatim Index Store Data of exactly such a word +/// keeps it, so a word outside the sampled set can round-trip and execute. A +/// word a guest builds with p[35:32] = 0xf has no flag and is decoded as an +/// ordinary word, whose classes 0x78-0x7f are unknown and refused. It is not a +/// claim about what hardware stores. +const L1I_RAW: u64 = 0xf; + +/// Bit 1 of the stored parity slot: the word is the model's raw form. Never +/// shown to the guest (Index Load Data returns bit 0 only). +const L1I_RAW_FLAG: u32 = 2; + +/// The instruction a staged word encodes, or the class that is refused. +fn l1i_decode(p: u64, raw_ok: bool) -> Result { + if raw_ok && (p >> 32) & 0xf == L1I_RAW { + return Ok(p as u32); + } + let class = (p >> 29) as u8 & 0x7f; + let row = L1I_CLASSES.iter().find(|c| c.class == class).ok_or(class)?; + let op = (p >> 26) as u8 & 7; + let funct = (p >> 20) as u8 & 0x3f; + // The class, p[28:26] and for the SPECIAL members the function field + // identify the instruction; anything else in a known class is refused. + let special = !row.funct.is_empty() && op == 0; + if !row.ops.contains(&op) || (special && !row.funct.contains(&funct)) || (class == 0 && p != 0) { + return Err(class); + } + let (hi3, rt_low) = (row.hi3, row.rt_low); + let (low, high) = ((p & 0x1f) as u32, ((p >> 5) & 0x1f) as u32); + let (rt, rd) = if rt_low { (low, high) } else { (high, low) }; + Ok(hi3 << 29 + | ((p >> 26) as u32 & 7) << 26 + | ((p >> 10) as u32 & 0x1f) << 21 + | rt << 16 + | rd << 11 + | ((p >> 15) as u32 & 0x1f) << 6 + | ((p >> 20) as u32 & 0x3f)) +} + +#[inline(always)] +fn l1i_tag_of(pa: u64) -> u64 { + (((pa >> 12) & 0xff_ffff) << 8) | (((pa >> 36) & 0xf) << 32) +} /// Does the secondary tag hold a valid line? Every state but Invalid does. #[inline(always)] @@ -166,7 +295,8 @@ struct Staged { /// The bus as a cached access sees it while staged lines are held: a staged /// secondary line answers for its addresses, memory for the rest. Put in /// front of the bus by `ShadowCache::serve_window`, and dropped with the last -/// line. +/// line. The one for instruction fetches (`fetch`) also finds staged primary +/// instruction lines. struct StagedBus< const IC_SIZE: usize, const IC_LINE: usize, const DC_SIZE: usize, const DC_LINE: usize, const L2_SIZE: usize, const L2_LINE: usize, const MIPS4: bool, const PRID: u32, @@ -176,6 +306,7 @@ struct StagedBus< /// its own `route`, and is not moved while the CPU runs. cache: *const ShadowCache, bus: Arc, + fetch: bool, } // Safety: reached only from the CPU thread, through the cache that owns it. @@ -227,6 +358,9 @@ impl< } } fn read32(&self, addr: u64) -> crate::traits::BusRead32 { + if self.fetch { + return self.cache().staged_fetch_read(addr); + } match self.cache().staged_read::<4>(addr) { Some(v) => crate::traits::BusRead32::ok(v as u32), None => self.bus.read32(addr), @@ -305,6 +439,7 @@ pub struct ShadowCache< /// front of it (`serve_window`). The bus paths make no test for staged /// lines; the object they call is what changes. route: UnsafeCell>, + fetch_route: UnsafeCell>, llbit: UnsafeCell, lladdr: UnsafeCell, /// CP0 ECC on the way in to a store, and on the way out of a load. @@ -338,6 +473,25 @@ pub struct ShadowCache< pd_tag_hook: UnsafeCell>>, /// Secondary lines the guest staged. R10000 family only; empty otherwise. staged: UnsafeCell, + /// Cache Error raised by the last fetch from a staged primary I word with + /// bad parity (`take_cache_err`). And the CPU's PC and Status registers + /// (`set_cpu_registers`): a fetch while lines are staged takes its virtual + /// address from the PC, and Status.DE and ERL turn the parity check off. + /// Read only on those paths, so the CPU tells the cache nothing as it runs. + l1i_err: UnsafeCell, + pc: UnsafeCell<*const u64>, + status: UnsafeCell<*const u32>, + /// Predecode classes already reported as refused, to warn once each. + l1i_refused: UnsafeCell>, + /// Raw-form words Index Load Data has returned: a verbatim Index Store + /// Data of one of these keeps the raw flag. At most `L1I_ISSUED_MAX` + /// words, emptied when full, and not saved in snapshots: the flag only + /// matters to a guest that copies a line it read out with Index Load Data + /// back with Index Store Data, which does so within the loop that reads + /// it. A word read before a snapshot and stored after it, or after 64 Ki + /// other distinct words, is stored without the flag and refused like any + /// guest-built 0xf word, reported on stderr, never run as raw bits. + l1i_issued: UnsafeCell>, } // Safety: the CPU thread is the only accessor, as for every other cache model @@ -378,6 +532,7 @@ impl< pub fn new(downstream: Arc) -> Self { Self { route: UnsafeCell::new(downstream.clone()), + fetch_route: UnsafeCell::new(downstream.clone()), downstream, llbit: UnsafeCell::new(false), lladdr: UnsafeCell::new(0), @@ -399,6 +554,11 @@ impl< decode: UnsafeCell::new(crate::cpu::decode_cache::DecodeCache::new()), pd_tag_hook: UnsafeCell::new(None), staged: UnsafeCell::new(Staged { valid: 0, changed: false }), + l1i_err: UnsafeCell::new(0), + pc: UnsafeCell::new(std::ptr::null()), + status: UnsafeCell::new(std::ptr::null()), + l1i_refused: UnsafeCell::new(Vec::new()), + l1i_issued: UnsafeCell::new(Default::default()), } } @@ -518,6 +678,10 @@ impl< let ecc = unsafe { *self.ecc_in.get() } & ecc_mask; s.data[slot] = phys_addr & data_mask; s.ecc[slot] = ecc; + if sel == CACH_PI && (s.data[slot] >> 32) & 0xf == L1I_RAW + && unsafe { (*self.l1i_issued.get()).contains(&s.data[slot]) } { + s.ecc[slot] |= L1I_RAW_FLAG; + } if matches!(sel, CACH_SI | CACH_SD) { // Index Store Data (S) writes a quadword: the selected // doubleword plus a zero partner, sharing the check bits @@ -543,17 +707,19 @@ impl< // doubleword of it reads as zero rather than as memory. // // Not modelled, and loudly absent rather than guessed: - // - the primary caches. A primary line the guest staged with the primary - // index operations is not consulted, so where one overlaps a staged - // secondary line the secondary data win, not the primary's; + // - the primary data cache. A line staged there with the primary index + // operations is not consulted by loads. (A staged primary instruction + // line is: see "Staged primary instruction-cache lines".) A valid + // primary instruction line is served before the secondary cache; // - replacement. An ordinary cached miss never fills, so never evicts: // a staged line lives until an index or hit operation, or an Index // Store Tag, ends it (`clanker/cpu/staged-secondary-cache-lines.md`); // - tag ECC and data check bits. Nothing is checked when a line hits, so // a line staged with bad check bits hits without a Cache Error; - // - the virtual-index field, TagLo[8:7]. A mismatch with the access's - // VA[13:12] purges primary lines at the recorded index, and there are - // no primary lines here to purge; + // - the virtual-index field, TagLo[8:7], as a check on a hit. A mismatch + // with the access's VA[13:12] purges primary lines at the recorded index + // and this model does not. The field is used when a secondary + // invalidate looks for primary lines (`l1i_purge_block`); // - a CACHE operand on an uncached page. It is undefined on the R10000; // the operation goes ahead as if the page were cached; // - the BadVAddr, Context and EntryHi values after a CACHE-raised TLB @@ -651,23 +817,48 @@ impl< /// paths read the fields this sets anyway, so the switch costs a normal /// access nothing. The decode cache forgets every page: its slots hold /// window pointers, or, after staging, words read through the bus that a - /// line may since have ended. + /// line may since have ended. Never reached during a fetch, which holds + /// the decode cache and its route: a fetch's refill only stages a line + /// beside the staged secondary line it reads. #[cold] #[inline(never)] fn serve_window(&self, hold: bool) { unsafe { *self.window_regions.get() = if hold { 0 } else { 64 }; *self.fetch_window.get() = if hold { std::ptr::null() } else { *self.tc_base.get() }; - *self.route.get() = if hold { - Arc::new(StagedBus { cache: self as *const Self, bus: self.downstream.clone() }) + if hold { + *self.route.get() = Arc::new(StagedBus { cache: self as *const Self, bus: self.downstream.clone(), fetch: false }); + *self.fetch_route.get() = Arc::new(StagedBus { cache: self as *const Self, bus: self.downstream.clone(), fetch: true }); } else { - self.downstream.clone() - }; + *self.route.get() = self.downstream.clone(); + *self.fetch_route.get() = self.downstream.clone(); + } (*self.staged.get()).changed = true; (*self.decode.get()).flush(); } } + /// The word at `a` for an instruction fetch while lines are staged, for + /// `StagedBus`: from a staged line if one holds it, else memory. Every + /// fetch is at the PC, and the word is in its page (an opcode-fused pair + /// never leaves its I-cache line), so its virtual address is the PC's page + /// and `a`'s offset. A cache with no CPU takes the virtual and physical + /// pages to be the same. + fn staged_fetch_read(&self, a: u64) -> crate::traits::BusRead32 { + let pc = unsafe { *self.pc.get() }; + let (va, fetched) = if pc.is_null() { + (a, true) + } else { + let pc = unsafe { *pc }; + ((pc & !0xfff) | (a & 0xfff), a & 0xfff == pc & 0xfff) + }; + match self.staged_fetch_word(va, a, fetched) { + Ok(Some(w)) => crate::traits::BusRead32::ok(w), + Ok(None) => self.downstream.read32(a), + Err(status) => crate::traits::BusRead32 { status, data: 0 }, + } + } + /// Store a secondary tag, keeping the staged-line count in step. fn set_l2_tag(&self, idx: usize, new: u64) { let l2 = unsafe { &mut *self.l2.get() }; @@ -679,6 +870,262 @@ impl< } } + // ---- Staged primary instruction-cache lines ------------------------- + // + // Index Store Tag (I) and Index Store Data (I) make a primary I line + // exactly as the secondary pair does, and a cached fetch that finds one + // takes its word from it before the secondary cache is consulted (NEC + // U10278EJ4V0UM pp.87-88). The primary index is the virtual address: + // VA[13:6] picks the set, VA[0] the way, VA[13:2] the word. The tag + // compares the fetch's physical address. Index Store Data (I) writes one + // word per operation, so a block takes 16. + // + // Model choices, each labelled by how well the sources settle it: + // - a fetch compares both ways at VA[13:6] against the fetch's translated + // PA[39:12]. Stated for Hit Invalidate (I), inferred for a fetch; + // - a fetch hit does not update LRU. Unstated, and no replacement is + // modelled; + // - the stored word is the 36-bit predecoded form, decoded on a fetch hit + // (`l1i_decode`) for the sampled classes only, per the datasheet's + // inferred layout. Any other class is refused: reported once, and the + // fetch is treated as a miss and reads the secondary cache or memory. + // APPROXIMATION, not hardware behavior: the sheet has a model refuse an + // unknown class, and hardware would run the word. Chosen over a Cache + // Error because memory holds the instruction the word was built from + // in all but a few stale entries, so a miss executes the right + // instruction for nearly every word a guest stages, where an error + // would stop a guest the hardware runs. The limit: a staged entry that + // is stale against memory runs the memory word, not the staged one. The + // once-only report keeps the refusal visible; + // - parity is checked on a fetch hit, even over the 36 bits and the parity + // bit. A bad word posts a Cache Error unless Status.DE or ERL is set; + // - Index Load Data (I) returns the stored word and parity as they were + // stored, so a readback stored back verbatim round-trips and executes. + // The only refill is from a staged secondary line (`l1i_refill`); other + // lines the guest never staged read back as the shadow's contents. Model + // choice, hardware contents unknown: a word with p[35:32] = 0xf is a raw + // instruction in p[31:0] (`L1I_RAW`); + // + // The primary data cache is not served: its tag and data layouts are in + // the sheet, but a dirty line writes back to the secondary cache and the + // modifier field has no model here. + + /// Physical address of the line a primary I `tag` at `set` names. + fn l1i_line_pa(&self, tag: u64, set: usize) -> u64 { + (((tag >> 8) & 0xff_ffff) << 12) + | (((tag >> 32) & 0xf) << 36) + | ((set as u64 * IC_LINE as u64) & 0xfff) + } + + /// Index Load Data handed out the raw word `p`. + fn l1i_note_issued(&self, p: u64) { + let issued = unsafe { &mut *self.l1i_issued.get() }; + if issued.len() >= L1I_ISSUED_MAX { + issued.clear(); + } + issued.insert(p); + } + + /// Test hook: stands for Index Load Data having handed out the raw word `p`. + #[cfg(test)] + pub fn issue_raw_for_test(&self, p: u64) { + self.l1i_note_issued(p); + } + + /// Store a primary I tag, keeping the staged-line count in step. + fn set_l1i_tag(&self, idx: usize, new: u64) { + let ic = unsafe { &mut *self.ic.get() }; + let old = ic.tags[idx]; + ic.tags[idx] = new; + let (was, now) = (old & L1I_VALID != 0, new & L1I_VALID != 0); + if was || now { + self.staged_move(was, now); + } + } + + /// The tag slot of the valid primary I line holding `pa` at the set `va` + /// indexes, if any. + fn staged_l1i_line(&self, va: u64, pa: u64) -> Option { + let ic = unsafe { &*self.ic.get() }; + if ic.data.is_empty() { + return None; + } + let set = (va as usize / IC_LINE) % (Self::IC_LINES / WAYS).max(1); + let want = l1i_tag_of(pa); + (0..WAYS) + .map(|way| set * WAYS + way) + .find(|&idx| ic.tags[idx] & L1I_VALID != 0 && (ic.tags[idx] ^ want) & L1I_TAG_PA_MASK == 0) + } + + /// The instruction a fetch at `va`/`pa` finds in a staged primary I line, + /// then (primary before secondary) in a staged secondary line. `Err` is a + /// Cache Error status. `fetched` is the word being fetched, as opposed to + /// the one after it that opcode fusion peeks at, which never raises. + #[inline(never)] + fn staged_fetch_word(&self, va: u64, pa: u64, fetched: bool) -> Result, u32> { + let mut hit = self.staged_l1i_line(va, pa); + if hit.is_none() && fetched && self.staged_any() && self.staged_slot(pa).is_some() { + // A primary miss on a staged secondary line: refill, then serve + // only from the way the refill filled. + hit = self.l1i_refill(va, pa); + } + if let Some(idx) = hit { + let ic = unsafe { &*self.ic.get() }; + let slot = (((va as usize) >> 2) % (ic.data.len() / WAYS)) * WAYS + idx % WAYS; + let (p, parity) = (ic.data[slot] & 0xf_ffff_ffff, ic.ecc[slot] & 1); + // Even parity over the 36 bits and the parity bit (NEC + // U10278EJ4V0UM pp.190-191). A bad word posts a Cache Error unless + // errors are not being detected (Status.DE, ERL). + if fetched && !l1i_parity_ok(p, parity) && self.errors_detected() { + // CacheErr.ED is the data-field error bit (UM p.265). The index + // fields follow the primary data check's: a model choice. + unsafe { *self.l1i_err.get() = L1I_CACHEERR_ED | (pa as u32 & 0x003f_fff8) | (va >> 12) as u32 & 7 }; + return Err(EXEC_CACHE_ERR); + } + match l1i_decode(p, ic.ecc[slot] & L1I_RAW_FLAG != 0) { + Ok(w) => return Ok(Some(w)), + Err(class) => { + // Refused: see `l1i_decode`. An approximation: the fetch is + // treated as a miss and reads the secondary cache or + // memory, and the class is reported once. A refusal never + // refills. + let seen = unsafe { &mut *self.l1i_refused.get() }; + if !seen.contains(&class) { + seen.push(class); + eprintln!("iris: staged primary I word {p:#011x} at {pa:#x}: predecode class {class:#04x} is not in the sampled set; fetching the staged secondary word, else memory, instead"); + } + } + } + } + Ok(self.staged_read::<4>(pa).map(|w| w as u32)) + } + + /// A cached fetch missed the primary I cache and hit a staged secondary + /// line: refill the 64-byte primary block from it (NEC U10278EJ4V0UM + /// pp.87-88: a primary instruction miss refills 64 bytes from the + /// secondary cache). Only this path fills a primary line; ordinary fetches + /// do not track residency. + /// Model choices: the victim is an Invalid way before a Valid one, and way + /// 0 when both are valid (the datasheet states no replacement for a refill); + /// the refilled words are stored in the model's raw encoding + /// (`L1I_RAW`), the architectural word carried in place of the predecoded + /// form hardware stores, with its even parity. + fn l1i_refill(&self, va: u64, pa: u64) -> Option { + let ic = unsafe { &mut *self.ic.get() }; + if ic.data.is_empty() { + return None; + } + let set = (va as usize / IC_LINE) % (Self::IC_LINES / WAYS).max(1); + let way = (0..WAYS).find(|&w| ic.tags[set * WAYS + w] & L1I_VALID == 0).unwrap_or(0); + let idx = set * WAYS + way; + let (vbase, pbase) = (va & !(IC_LINE as u64 - 1), pa & !(IC_LINE as u64 - 1)); + let per_way = ic.data.len() / WAYS; + for i in 0..(IC_LINE / 4) as u64 { + let w = self.staged_read::<4>(pbase + 4 * i).unwrap_or(0); + let p = L1I_RAW << 32 | w; + let slot = (((vbase >> 2) + i) as usize % per_way) * WAYS + way; + ic.data[slot] = p; + ic.ecc[slot] = (p.count_ones() & 1) | L1I_RAW_FLAG; + } + self.set_l1i_tag(idx, l1i_tag_of(pa) | L1I_VALID); + Some(idx) + } + + /// The valid primary lines, as (tag slot, physical address), whose block + /// lies in `[pbase, pbase + len)` and whose set index has VA[13:12] = + /// `pidx`. A primary line sits in the set its virtual address picked, so + /// the secondary tag's recorded index names the only sets it can be in: + /// the lookup is by set, not a scan of every tag. Both primary tags keep + /// PA[35:12] in bits 31:8 and PA[39:36] in bits 35:32. + fn l1_lines_in_block( + tags: &[u64], line: usize, pbase: u64, len: u64, pidx: u64, valid: impl Fn(u64) -> bool, + ) -> Vec<(usize, u64)> { + let sets = (tags.len() / WAYS).max(1); + let mut found = Vec::new(); + let mut a = pbase; + while a < pbase + len { + let set = ((((pidx << 12) | (a & 0xfff)) as usize) / line) % sets; + for idx in (0..WAYS).map(|way| set * WAYS + way) { + if valid(tags[idx]) && (tags[idx] ^ l1i_tag_of(a)) & L1I_TAG_PA_MASK == 0 { + found.push((idx, a)); + } + } + a += line as u64; + } + found + } + + /// Did the model refill the primary I line at tag slot `idx`? Its words + /// carry the raw flag (`l1i_refill`), and a guest that stored over the + /// first word has made the line its own. + fn l1i_refilled(&self, idx: usize) -> bool { + let ic = unsafe { &*self.ic.get() }; + let (set, way) = (idx / WAYS, idx % WAYS); + let slot = ((set * IC_LINE / 4) % (ic.data.len() / WAYS).max(1)) * WAYS + way; + ic.ecc.get(slot).is_some_and(|e| e & L1I_RAW_FLAG != 0) + } + + /// Invalidating a secondary block invalidates the primary instruction + /// lines it holds (NEC U12754EJ1V0UMJ1 p.17, 19), those at the virtual + /// index the secondary tag records (`pidx`, TagLo[8:7]). A line the model + /// refilled itself goes whatever its index: a refill follows the fetch's + /// VA[13:12], where hardware would have recorded that index in the tag, + /// and the model keeps the guest's staged tag as it is. + fn l1i_purge_block(&self, pbase: u64, len: u64, pidx: u64) { + let ic = unsafe { &*self.ic.get() }; + if IC_LINE == 0 || ic.tags.is_empty() { + return; + } + for index in 0..4 { + for (idx, _) in Self::l1_lines_in_block(&ic.tags, IC_LINE, pbase, len, index, |t| t & L1I_VALID != 0) { + if index == pidx || self.l1i_refilled(idx) { + self.l1i_invalidate(idx); + } + } + } + } + + /// The same for the primary data lines: every line whose state is not + /// Invalid goes Invalid with its state modifier reset. A line written + /// and inconsistent with the secondary cache (modifier 010) is first + /// written back into the secondary line at tag slot `l2_idx` when + /// `writeback` (Index and Hit Writeback Invalidate; Hit Invalidate does + /// not). The primary data cache is otherwise unmodelled: its tags and + /// data are the diagnostic shadow, so this is all the inclusion rule + /// reaches there. + fn l1d_purge_block(&self, l2_idx: usize, pbase: u64, len: u64, pidx: u64, writeback: bool) { + let (dc, l2) = unsafe { (&mut *self.dc.get(), &mut *self.l2.get()) }; + if DC_LINE == 0 || dc.tags.is_empty() { + return; + } + let lines = Self::l1_lines_in_block(&dc.tags, DC_LINE, pbase, len, pidx, |t| (t >> L1D_STATE_SHIFT) & 3 != 0); + for (idx, pa) in lines { + let tag = dc.tags[idx]; + if writeback && (tag >> L1D_MODIFIER_SHIFT) & 7 == L1D_MODIFIER_WRITTEN + && !dc.data.is_empty() && !l2.data.is_empty() { + let (set, way) = (idx / WAYS, idx % WAYS); + let per_way = l2.data.len() / WAYS; + let (l2_set, l2_way) = (l2_idx / WAYS, l2_idx % WAYS); + for d in 0..DC_LINE / 8 { + // Word `k` of this doubleword, by the data array's own slot + // function; the word at the lower address is TagLo. + let word = |k: usize| dc.data[self.data_slot(CACH_PD, dc.data.len(), (set * DC_LINE + 8 * d + 4 * k) as u64 | way as u64)] & 0xffff_ffff; + let i = ((pa % L2_LINE as u64) / 8) as usize + d; + l2.data[((l2_set * L2_LINE / 8 + i) % per_way) * WAYS + l2_way] = word(1) << 32 | word(0); + } + } + dc.tags[idx] = tag & !(3 << L1D_STATE_SHIFT) & !(7 << L1D_MODIFIER_SHIFT); + } + } + + /// Clear the valid bit of the primary I line at `idx`. + fn l1i_invalidate(&self, idx: usize) { + let tag = unsafe { (*self.ic.get()).tags[idx] }; + if tag & L1I_VALID != 0 { + self.set_l1i_tag(idx, tag & !L1I_VALID); + } + } + /// Physical address of the line a secondary `tag` at `set` names: the /// tag's PA[39:18] and the set's own PA[17:B] below it. fn l2_line_pa(&self, tag: u64, set: usize) -> u64 { @@ -699,10 +1146,15 @@ impl< if !l2_tag_valid(tag) { return; } + // The primary lines are located by the stored tag's physical block and + // its recorded virtual index, not by the operand (ISUM pp.19, 27, 28). + let (base, pidx) = (self.l2_line_pa(tag, idx / WAYS), (tag >> L2_VINDEX_SHIFT) & 3); + self.l1d_purge_block(idx, base, L2_LINE as u64, pidx, writeback); if writeback && (tag >> L2_STATE_SHIFT) & 3 == L2_STATE_DIRTY_EXCLUSIVE { self.l2_write_line_back(idx); } self.set_l2_tag(idx, tag & !(3 << L2_STATE_SHIFT)); + self.l1i_purge_block(base, L2_LINE as u64, pidx); } /// Send the block at tag slot `idx` to the system interface, doubleword by @@ -736,20 +1188,40 @@ impl< } } - /// Test hook: does a valid staged line lie in the page holding `phys`? + /// Test hook: does a valid staged line, secondary or primary I, lie in + /// the page holding `phys`? #[cfg(test)] pub fn page_staged(&self, phys: u64) -> bool { - let l2 = unsafe { &*self.l2.get() }; - l2.tags.iter().enumerate().any(|(i, &t)| l2_tag_valid(t) && self.l2_line_pa(t, i / WAYS) >> 12 == phys >> 12) + let (l2, ic) = unsafe { (&*self.l2.get(), &*self.ic.get()) }; + let page = phys >> 12; + l2.tags.iter().enumerate().any(|(i, &t)| l2_tag_valid(t) && self.l2_line_pa(t, i / WAYS) >> 12 == page) + || ic.tags.iter().enumerate().any(|(i, &t)| t & L1I_VALID != 0 && self.l1i_line_pa(t, i / WAYS) >> 12 == page) + } + + /// Are cache errors detected: Status has neither DE nor ERL? A cache with + /// no CPU behind it detects them. + fn errors_detected(&self) -> bool { + use crate::cpu::mips_core::{STATUS_DE, STATUS_ERL}; + let status = unsafe { *self.status.get() }; + status.is_null() || unsafe { *status } & (STATUS_DE | STATUS_ERL) == 0 } - /// Index Writeback Invalidate, Hit Invalidate and Hit Writeback Invalidate - /// (S) while lines are staged. Other caches' forms do nothing here. + /// Index Writeback Invalidate (S), Index Invalidate (I), Hit Invalidate + /// (I, S) and Hit Writeback Invalidate (S) while lines are staged. Index + /// Invalidate (I) clears the valid bit of the way named, Hit Invalidate + /// (I) of whichever way holds the address (NEC U12754EJ1V0UMJ1 pp.18, + /// 27). Other caches' forms do nothing here. #[cold] #[inline(never)] fn end_staged_line(&self, op: u32, sel: u32, virt_addr: u64, phys_addr: u64) { match (op, sel) { (C_IWBINV, CACH_SD) => self.l2_invalidate(self.tag_slot(sel, virt_addr), true), + (C_IINV, CACH_PI) => self.l1i_invalidate(self.tag_slot(sel, virt_addr)), + (C_HINV, CACH_PI) => { + if let Some(idx) = self.staged_l1i_line(virt_addr, phys_addr) { + self.l1i_invalidate(idx); + } + } (C_HINV, CACH_SD) => self.l2_hit_invalidate(phys_addr, false), (C_HWBINV, CACH_SD) => self.l2_hit_invalidate(phys_addr, true), _ => {} @@ -884,14 +1356,14 @@ impl< } #[inline(always)] - fn fetch(&self, _virt_addr: u64, phys_addr: u64) -> FetchInstrResult { + fn fetch(&self, virt_addr: u64, phys_addr: u64) -> FetchInstrResult { // Decoded instructions by physical page — see `decode_cache`. let dc = unsafe { &mut *self.decode.get() }; // Only a word the decode cache cannot take from the window comes // here. While lines are staged that is every word (`fetch_window`), // and the route serves the staged lines (`StagedBus`). let word = |a: u64| { - let r = unsafe { &*self.route.get() }.read32(a); + let r = unsafe { &*self.fetch_route.get() }.read32(a); if r.is_ok() { Ok(r.data) } else { Err(r.status) } }; let window = if Self::R10K { unsafe { *self.fetch_window.get() } } else { unsafe { *self.tc_base.get() } }; @@ -1075,6 +1547,8 @@ impl< let tag = phys_addr & mask & !MRU_BIT; if Self::R10K && sel == CACH_SD { self.set_l2_tag(idx, tag); + } else if Self::R10K && sel == CACH_PI { + self.set_l1i_tag(idx, tag); } else { s.tags[idx] = tag; } @@ -1108,7 +1582,16 @@ impl< if s.data.is_empty() { 0 } else { - unsafe { *self.ecc_out.get() = s.ecc[slot] }; + let mut ecc = s.ecc[slot]; + if sel == CACH_PI { + // The raw flag is the model's, not the guest's. A raw + // word handed out can be stored back verbatim. + if ecc & L1I_RAW_FLAG != 0 { + self.l1i_note_issued(s.data[slot]); + } + ecc &= 1; + } + unsafe { *self.ecc_out.get() = ecc }; if l2_op_log(CACHE_LOG_HIT) { crate::dlog!(LogModule::L2c, "shadow: -> ILD slot={slot} data={:#018x} ecc={:#x}", s.data[slot], s.ecc[slot]); @@ -1130,7 +1613,8 @@ impl< 0 } - // The operations that end a staged line. With none staged they + // The operations that end a staged line (Index Invalidate (I) has + // Index Writeback Invalidate's encoding). With none staged they // have nothing to end, since a valid tag is a staged line, so they // and every other cache's fall through: they move data between a // cache and memory, and here those never disagree. @@ -1158,6 +1642,7 @@ impl< st.valid = 0; self.serve_window(false); } + unsafe { (*self.l1i_issued.get()).clear() }; unsafe { (*self.decode.get()).flush() }; } @@ -1189,45 +1674,107 @@ impl< lines.push(toml::Value::Table(e)); } t.insert("staged_l2".into(), toml::Value::Array(lines)); + let ic = unsafe { &*self.ic.get() }; + let per_way = (ic.data.len() / WAYS).max(1); + let mut ilines = Vec::new(); + for (idx, &tag) in ic.tags.iter().enumerate().filter(|(_, &t)| t & L1I_VALID != 0) { + let (set, way) = (idx / WAYS, idx % WAYS); + let words = (0..IC_LINE / 4).map(|i| toml::Value::Integer(ic.data[((set * IC_LINE / 4 + i) % per_way) * WAYS + way] as i64)); + let mut e = toml::value::Table::new(); + e.insert("idx".into(), toml::Value::Integer(idx as i64)); + e.insert("tag".into(), u(tag)); + e.insert("words".into(), toml::Value::Array(words.collect())); + let parity = (0..IC_LINE / 4).map(|i| toml::Value::Integer(ic.ecc[((set * IC_LINE / 4 + i) % per_way) * WAYS + way] as i64)); + e.insert("parity".into(), toml::Value::Array(parity.collect())); + ilines.push(toml::Value::Table(e)); + } + t.insert("staged_l1i".into(), toml::Value::Array(ilines)); toml::Value::Table(t) } /// A restore replaces RAM wholesale: drop every decoded instruction, end /// every staged line and bring back the saved ones. fn load_cache_state(&self, v: &toml::Value) -> Result<(), String> { + let u = |v: &toml::Value| -> Result { + let a = v.as_array().filter(|a| a.len() == 2).ok_or("staged line: bad doubleword")?; + let w = |x: &toml::Value| x.as_integer().map(|i| i as u64 & 0xffff_ffff).ok_or("staged line: bad word"); + Ok(w(&a[0])? | w(&a[1])? << 32) + }; + let int = |v: Option<&toml::Value>, what: &'static str| v.and_then(|x| x.as_integer()).ok_or(what); + // Phase 1: parse and check everything, touching nothing, so a bad + // snapshot leaves the cache as it was. + struct L2Line { idx: usize, tag: u64, mru: bool, data: Vec, ecc: Vec } + struct L1iLine { idx: usize, tag: u64, words: Vec, parity: Vec } + let (mut l2s, mut l1s) = (Vec::new(), Vec::new()); + if Self::R10K { + let (l2, ic) = unsafe { (&*self.l2.get(), &*self.ic.get()) }; + for e in v.get("staged_l2").and_then(|l| l.as_array()).into_iter().flatten() { + let idx = int(e.get("idx"), "staged line: no index")? as usize; + if idx >= l2.tags.len() || l2.data.is_empty() { + return Err(format!("staged line index {idx} outside the secondary cache")); + } + let data = e.get("data").and_then(|d| d.as_array()).ok_or("staged line: no data")?; + let ecc = e.get("ecc").and_then(|d| d.as_array()).ok_or("staged line: no check bits")?; + if data.len() != L2_LINE / 8 || ecc.len() != data.len() { + return Err("staged line: wrong size for this cache".into()); + } + l2s.push(L2Line { + idx, + tag: u(e.get("tag").ok_or("staged line: no tag")?)?, + mru: e.get("mru").and_then(|m| m.as_bool()).unwrap_or(false), + data: data.iter().map(&u).collect::>()?, + ecc: ecc.iter().map(|c| c.as_integer().map(|i| i as u32).ok_or("staged line: bad check bits")).collect::>()?, + }); + } + for e in v.get("staged_l1i").and_then(|l| l.as_array()).into_iter().flatten() { + let idx = int(e.get("idx"), "staged line: no index")? as usize; + let words = e.get("words").and_then(|d| d.as_array()).ok_or("staged line: no words")?; + let parity = e.get("parity").and_then(|d| d.as_array()).ok_or("staged line: no parity")?; + if idx >= ic.tags.len() || words.len() != IC_LINE / 4 || parity.len() != words.len() || ic.data.is_empty() { + return Err("staged primary line does not fit this cache".into()); + } + l1s.push(L1iLine { + idx, + tag: u(e.get("tag").ok_or("staged line: no tag")?)?, + words: words.iter().map(|w| w.as_integer().map(|i| i as u64).ok_or("staged line: bad word")).collect::>()?, + parity: parity.iter().map(|x| x.as_integer().map(|i| i as u32).ok_or("staged line: bad parity")).collect::>()?, + }); + } + } + // Phase 2: end every staged line, then bring back the saved ones. let l2 = unsafe { &mut *self.l2.get() }; + let ic = unsafe { &mut *self.ic.get() }; if Self::R10K { for idx in 0..l2.tags.len() { if l2_tag_valid(l2.tags[idx]) { self.set_l2_tag(idx, l2.tags[idx] & !(3 << L2_STATE_SHIFT)); } } - } - let lines = v.get("staged_l2").and_then(|l| l.as_array()); - let u = |v: &toml::Value| -> Result { - let a = v.as_array().filter(|a| a.len() == 2).ok_or("staged line: bad doubleword")?; - let w = |x: &toml::Value| x.as_integer().map(|i| i as u64 & 0xffff_ffff).ok_or("staged line: bad word"); - Ok(w(&a[0])? | w(&a[1])? << 32) - }; - for e in lines.into_iter().flatten().filter(|_| Self::R10K) { - let idx = e.get("idx").and_then(|i| i.as_integer()).ok_or("staged line: no index")? as usize; - if idx >= l2.tags.len() || l2.data.is_empty() { - return Err(format!("staged line index {idx} outside the secondary cache")); + for idx in 0..ic.tags.len() { + self.l1i_invalidate(idx); } - let (set, way) = (idx / WAYS, idx % WAYS); + unsafe { (*self.l1i_issued.get()).clear() }; + } + for e in l2s { + let (set, way) = (e.idx / WAYS, e.idx % WAYS); let per_way = l2.data.len() / WAYS; - let data = e.get("data").and_then(|d| d.as_array()).ok_or("staged line: no data")?; - let ecc = e.get("ecc").and_then(|d| d.as_array()).ok_or("staged line: no check bits")?; - if data.len() != L2_LINE / 8 || ecc.len() != data.len() { - return Err("staged line: wrong size for this cache".into()); - } - for (i, (d, c)) in data.iter().zip(ecc).enumerate() { + for (i, (d, c)) in e.data.iter().zip(&e.ecc).enumerate() { let slot = ((set * L2_LINE / 8 + i) % per_way) * WAYS + way; - l2.data[slot] = u(d)?; - l2.ecc[slot] = c.as_integer().ok_or("staged line: bad check bits")? as u32; + l2.data[slot] = *d; + l2.ecc[slot] = *c; + } + l2.mru[set.min(l2.mru.len() - 1)] = e.mru; + self.set_l2_tag(e.idx, e.tag); + } + for e in l1s { + let (set, way) = (e.idx / WAYS, e.idx % WAYS); + let per_way = ic.data.len() / WAYS; + for (i, (w, p)) in e.words.iter().zip(&e.parity).enumerate() { + let slot = ((set * IC_LINE / 4 + i) % per_way) * WAYS + way; + ic.data[slot] = *w; + ic.ecc[slot] = *p; } - l2.mru[set.min(l2.mru.len() - 1)] = e.get("mru").and_then(|m| m.as_bool()).unwrap_or(false); - self.set_l2_tag(idx, u(e.get("tag").ok_or("staged line: no tag")?)?); + self.set_l1i_tag(e.idx, e.tag); } unsafe { (*self.decode.get()).flush() }; Ok(()) @@ -1249,7 +1796,11 @@ impl< /// `l2 probe` in the monitor: how many secondary lines are staged. fn debug_probe(&self, _cache_name: &str, _virt_addr: u64, _phys_addr: u64) -> String { let st = unsafe { &*self.staged.get() }; - format!("staged secondary lines: {}", st.valid) + let l2 = unsafe { &*self.l2.get() }; + let ic = unsafe { &*self.ic.get() }; + let sec = l2.tags.iter().filter(|&&t| l2_tag_valid(t)).count(); + let pri = ic.tags.iter().filter(|&&t| t & L1I_VALID != 0).count(); + format!("staged lines: {} (secondary {sec}, primary I {pri})", st.valid) } fn get_config(&self, cache_target: u32) -> (usize, usize) { @@ -1270,6 +1821,17 @@ impl< unsafe { *self.ecc_out.get() } } + unsafe fn set_cpu_registers(&self, pc: *const u64, status: *const u32) { + unsafe { + *self.pc.get() = pc; + *self.status.get() = status; + } + } + + fn take_cache_err(&self) -> u32 { + if Self::R10K { std::mem::take(unsafe { &mut *self.l1i_err.get() }) } else { 0 } + } + fn downstream(&self) -> Arc { self.downstream.clone() } @@ -2106,42 +2668,6 @@ mod tests { check::("R12000 2 MiB/128"); } - /// The set of a secondary line is PA[N-2:B]. The geometry test's lines all - /// have the top index bit clear, which a lookup fixed at 4096 sets also - /// passes. Set the top bit (and the one below it) at every cache size and - /// find, load, hit-invalidate and index-invalidate the line. - #[test] - fn the_top_secondary_index_bit_reaches_the_line_at_every_cache_size() { - fn check>>(name: &str) { - let line = C::L2_LINE as u64; - let way_bytes = (C::L2_SIZE / 2) as u64; - let top = way_bytes >> 1; - for a in [0x0020_3000u64 + top, 0x0020_3000 + (top >> 1), 0x0020_3000 + top + (top >> 1)] { - let bus = Arc::new(Bus::default()); - let c = C::from(bus.clone() as Arc); - let op = |pa: u64, way: u64| 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8) | way; - let tag = |pa: u64, way: u64, state: u64| { - c.cache_op(C_IST | CACH_SD, op(pa & !(line - 1), way), l2_tag_of(pa) | state << L2_STATE_SHIFT); - }; - let load = |pa: u64| c.read::<8>(op(pa, 0), pa).data; - tag(a, 1, 2); - c.cache_op(C_R10K_ISD | CACH_SD, op(a & !7, 1), 0x0a0a_0a0a_0b0b_0b0b); - assert_eq!(load(a), 0x0b0b_0b0b_0a0a_0a0a, "{name}: {a:#x} hits"); - assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "{name}: {a:#x}"); - // The line of the same set without the top bit is another set. - assert_eq!(load(a ^ top), MEM, "{name}: {a:#x} with the top bit flipped"); - c.cache_op(C_HINV | CACH_SD, op(a, 0), a); - assert_eq!(load(a), MEM, "{name}: {a:#x} Hit Invalidate ends it"); - tag(a, 1, 2); - c.cache_op(C_IWBINV | CACH_SD, op(a, 1), 0); - assert_eq!(load(a), MEM, "{name}: {a:#x} Index Writeback Invalidate ends it"); - } - } - check::("R10000 1 MiB/128"); - check::("R10000 O2 1 MiB/64"); - check::("R12000 2 MiB/128"); - } - /// A snapshot carries the staged lines, and a restore ends any that were /// staged before it. Dirty lines also reach memory on `writeback_all`. #[test] @@ -2190,6 +2716,8 @@ mod tests { let op = |pa: u64| 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8); let quads = line / 16; for target in 0..quads { + // End the previous round's line (and with it any primary refill). + c.cache_op(C_IWBINV | CACH_SD, op(pa0), 0); c.cache_op(C_IST | CACH_SD, op(pa0), l2_tag_of(pa0) | 2 << L2_STATE_SHIFT); // Clear the whole line, then store the target quadword once. for q in 0..quads { @@ -2231,6 +2759,754 @@ mod tests { check::("64-byte lines"); } + // ---- Staged primary instruction-cache lines ------------------------- + + /// Index Store Tag (I) on `pa`'s block at the set `va` indexes, `way`. + fn stage_i_tag(c: &R10000ShadowCache, va: u64, pa: u64, way: u64, valid: bool) { + let tag = l1i_tag_of(pa) | if valid { L1I_VALID } else { 0 }; + c.cache_op(C_IST | CACH_PI, (va & !63) | way, tag); + } + + /// Index Store Data (I): the word at `va`, in `way`, is the 36-bit `p`, + /// with the parity bit that makes it even. + fn stage_p(c: &R10000ShadowCache, va: u64, way: u64, p: u64) { + // A raw word in a test stands for one Index Load Data handed out. + if (p >> 32) & 0xf == L1I_RAW { + unsafe { (*c.l1i_issued.get()).insert(p) }; + } + c.set_cache_ecc(l1i_parity_of(p)); + c.cache_op(C_R10K_ISD | CACH_PI, (va & !3) | way, p); + } + + /// The same for a plain instruction, in the model's raw encoding. + fn stage_i_word(c: &R10000ShadowCache, va: u64, way: u64, w: u32) { + stage_p(c, va, way, L1I_RAW << 32 | w as u64); + } + + /// A fetch at `va`, which the cache takes as the PC. + fn fetch_raw(c: &R10000ShadowCache, va: u64, pa: u64) -> u32 { + let pc = va; + unsafe { c.set_cpu_registers(&pc, std::ptr::null()) }; + let raw = unsafe { (*c.fetch(va, pa).instr).raw }; + unsafe { c.set_cpu_registers(std::ptr::null(), std::ptr::null()) }; + raw + } + + /// Stage whole primary I blocks on way 1 of several sets (16 words each) + /// and fetch through them: the staged words, with no bus read. + #[test] + fn a_staged_primary_instruction_line_serves_fetches() { + let (c, bus) = staged_cache(); + let base = 0x0030_0000u64; + for set in 0..4u64 { + let (va, pa) = (0xffff_ffff_8000_0000 | (base + set * 64), base + set * 64); + stage_i_tag(&c, va, pa, 1, true); + for w in 0..16u64 { + stage_i_word(&c, va + 4 * w, 1, 0x3c00_0000 | (set << 8) as u32 | w as u32); + } + } + for set in 0..4u64 { + for w in 0..16u64 { + let (va, pa) = (0xffff_ffff_8000_0000 | (base + set * 64 + 4 * w), base + set * 64 + 4 * w); + assert_eq!(fetch_raw(&c, va, pa), 0x3c00_0000 | (set << 8) as u32 | w as u32, "set {set} word {w}"); + } + } + assert_eq!(bus.reads.load(Ordering::Relaxed), 0); + // The index is virtual: the same physical block through a VA of + // another page colour (VA[13:12]) misses. The tag is physical: another + // frame misses. + assert_eq!(fetch_raw(&c, 0xffff_ffff_8000_0000 | (base + 0x1000), base), 0xeeee_eeee); + assert_eq!(fetch_raw(&c, 0xffff_ffff_8000_0000 | base, base + (1 << 20)), 0xeeee_eeee); + // Loads are not served by an instruction line. + assert_eq!(load::<4>(&c, base), 0xeeee_eeee); + } + + /// A secondary staged line served fetches first; ending it leaves the + /// primary I lines hitting. With both valid the primary wins. + #[test] + fn primary_instruction_lines_outlive_the_secondary_line_and_win_over_it() { + let (c, bus) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_tag(&c, pa, 0, 2); + for q in 0..8 { + stage_data(&c, pa + 16 * q, 0, 0x5555_0000_6666_0000 | q); + } + assert_eq!(fetch_raw(&c, va, pa), 0x6666_0000, "the secondary line"); + // The fetch above refilled way 0; stage over it. + stage_i_tag(&c, va, pa, 0, true); + for w in 0..16 { + stage_i_word(&c, va + 4 * w, 0, 0x7000 + w as u32); + } + assert_eq!(fetch_raw(&c, va, pa), 0x7000, "primary before secondary"); + stage_tag(&c, pa, 0, 0); // the secondary line ends + assert_eq!(fetch_raw(&c, va, pa), 0x7000); + assert_eq!(fetch_raw(&c, va + 8, pa + 8), 0x7002); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0); + } + + /// Ending a primary I line: Index Store Tag with an invalid state, Index + /// Invalidate (the way named), Hit Invalidate (by address), power-on. A + /// fetch served before must not be served again. Another way, another + /// set, and an Index Invalidate of the other way leave the line. + #[test] + fn primary_instruction_lines_end() { + let (pa, va) = (0x0030_0040u64, 0xffff_ffff_8030_0040u64); + let ends: [(&str, Box); 4] = [ + ("Index Store Tag, invalid", Box::new(move |c| stage_i_tag(c, va, pa, 1, false))), + ("Index Invalidate", Box::new(move |c| { c.cache_op(C_IINV | CACH_PI, va | 1, 0); })), + ("Hit Invalidate", Box::new(move |c| { c.cache_op(C_HINV | CACH_PI, va, pa); })), + ("power-on", Box::new(|c| c.power_on())), + ]; + for (name, end) in ends { + let (c, _) = staged_cache(); + stage_i_tag(&c, va, pa, 1, true); + stage_i_word(&c, va, 1, 0x1234); + assert_eq!(fetch_raw(&c, va, pa), 0x1234, "{name}: before"); + assert!(c.page_staged(pa)); + end(&c); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee, "{name}: after"); + assert!(!c.page_staged(pa), "{name}"); + } + let (c, _) = staged_cache(); + stage_i_tag(&c, va, pa, 1, true); + stage_i_word(&c, va, 1, 0x1234); + c.cache_op(C_IINV | CACH_PI, va, 0); // the other way + c.cache_op(C_IINV | CACH_PI, va + 64, 0); // the next set + c.cache_op(C_HINV | CACH_PI, va, pa + (1 << 20)); // misses + assert_eq!(fetch_raw(&c, va, pa), 0x1234); + } + + /// Decode of the sampled class against the datasheet's example: + /// `addiu $4,$0,2` (0x24040002) is 0x3_4420_0004. Each field moves where + /// the layout says, so a change to any one fails. + #[test] + fn a_sampled_predecoded_word_decodes_to_its_instruction() { + assert_eq!(l1i_decode(0x3_4420_0004, false), Ok(0x2404_0002)); + // addiu $17,$9,-1: rs, rt, immediate and shamt/rd bits all move. + let f = 0x2531_ffffu32; // op 001001, rs 9, rt 17, imm 0xffff + let p = (0x1au64 << 29) | ((f as u64 >> 26) & 7) << 26 | (f as u64 & 0x3f) << 20 + | ((f as u64 >> 6) & 0x1f) << 15 | ((f as u64 >> 21) & 0x1f) << 10 + | ((f as u64 >> 11) & 0x1f) << 5 | ((f as u64 >> 16) & 0x1f); + assert_eq!(l1i_decode(p, false), Ok(f)); + assert_eq!(l1i_decode(0x3_4420_0004 | 0x1f << 5, false), Ok(0x2404_0002 | 0x1f << 11), "p[9:5] is the rd position for addiu"); + assert_eq!(l1i_decode(0x3_4420_0004 | 0x1f << 10, false), Ok(0x2404_0002 | 0x1f << 21), "p[14:10] is rs"); + assert_eq!(l1i_decode(0x3_4420_0004 | 0x1f << 15, false), Ok(0x2404_0002 | 0x1f << 6), "p[19:15] is f[10:6]"); + } + + /// One example per class of the table, built from the instruction by the + /// layout, decoded back: includes dsll/dsll32, jr/j and the nop. + #[test] + fn every_class_in_the_table_decodes() { + fn predecode(class: u64, f: u32, rt_low: bool) -> u64 { + let (rt, rd) = ((f as u64 >> 16) & 0x1f, (f as u64 >> 11) & 0x1f); + let (lo, hi) = if rt_low { (rt, rd) } else { (rd, rt) }; + class << 29 | ((f as u64 >> 26) & 7) << 26 | (f as u64 & 0x3f) << 20 + | ((f as u64 >> 6) & 0x1f) << 15 | ((f as u64 >> 21) & 0x1f) << 10 | hi << 5 | lo + } + let cases: &[(u64, u32, bool, &str)] = &[ + (0x08, 0x0009_20f8, false, "dsll $4,$9,3"), // funct 0x38 + (0x08, 0x0009_20fc, false, "dsll32 $4,$9,3"), // funct 0x3c + (0x18, 0x0109_5025, false, "or $10,$8,$9"), + (0x1a, 0x2084_0010, true, "addi"), + (0x1a, 0x2484_0010, true, "addiu"), + (0x1a, 0x3484_0010, true, "ori"), + (0x1a, 0x3c04_8000, true, "lui"), + (0x1e, 0x6484_0010, true, "daddiu"), + (0x35, 0xbd04_0040, false, "cache"), + (0x36, 0xdd04_0008, true, "ld"), + (0x3f, 0x4084_6000, false, "mtc0"), + (0x40, 0x0c00_1234, false, "jal"), + (0x41, 0x03e0_0008, false, "jr $31"), + (0x41, 0x0800_1234, false, "j"), + (0x71, 0x0481_0004, false, "bgez"), + ]; + for &(class, f, rt_low, name) in cases { + assert_eq!(l1i_decode(predecode(class, f, rt_low), false), Ok(f), "{name}"); + } + assert_eq!(l1i_decode(0, false), Ok(0), "the nop"); + assert_eq!(l1i_decode(0x3_4420_0004, false), Ok(0x2404_0002)); + // Known class, instruction outside its rows: refused. + assert_eq!(l1i_decode(predecode(0x08, 0x0009_20c0, false), false), Err(0x08), "funct 0x00 is not dsll"); + assert_eq!(l1i_decode(predecode(0x1a, 0x2884_0010, true), false), Err(0x1a), "op 0x0a is not in 0x1a"); + assert_eq!(l1i_decode(1, false), Err(0), "class 0 is only the nop"); + // Classes share an opcode top: the class picks f[31:29]. + assert_ne!(l1i_decode(predecode(0x1a, 0x2484_0010, true), false), l1i_decode(predecode(0x1e, 0x2484_0010, true), false)); + } + + /// A class the sheet does not sample is refused, not guessed: the fetch is + /// treated as a miss and reads memory, and the refusal is reported. + #[test] + fn an_unsampled_class_is_refused_and_the_fetch_reads_memory() { + assert_eq!(l1i_decode(0x5_0000_0004, false), Err(0x28)); + assert_eq!(l1i_decode(0x2a << 29, false), Err(0x2a)); + let (c, bus) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_i_tag(&c, va, pa, 1, true); + stage_p(&c, va, 1, 0x5_0000_0004); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee); + assert!(bus.reads.load(Ordering::Relaxed) >= 1, "memory was read"); + // The sampled class beside it still executes. + stage_p(&c, va + 4, 1, 0x3_4420_0004); + assert_eq!(fetch_raw(&c, va + 4, pa + 4), 0x2404_0002); + } + + /// A word staged with the wrong parity bit posts a Cache Error on fetch + /// (ED set), unless Status.DE or ERL is set, which executes it. + #[test] + fn bad_parity_posts_a_cache_error_unless_errors_are_not_detected() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_i_tag(&c, va, pa, 1, true); + c.set_cache_ecc(l1i_parity_of(0x3_4420_0004) ^ 1); + c.cache_op(C_R10K_ISD | CACH_PI, va | 1, 0x3_4420_0004); + let r = c.fetch(va, pa); + assert_eq!(r.status, EXEC_CACHE_ERR); + assert_ne!(c.take_cache_err() & L1I_CACHEERR_ED, 0); + assert_eq!(c.take_cache_err(), 0, "reading clears it"); + let mut status = crate::cpu::mips_core::STATUS_DE; + unsafe { c.set_cpu_registers(std::ptr::null(), &status) }; + let r = c.fetch(va, pa); + assert_eq!(r.status, crate::cpu::mips_exec::EXEC_COMPLETE); + assert_eq!(unsafe { (*r.instr).raw }, 0x2404_0002); + // Every single-bit flip of the 36 bits is a parity error. + status = 0; + unsafe { c.set_cpu_registers(std::ptr::null(), &status) }; + for bit in 0..36 { + c.set_cache_ecc(l1i_parity_of(0x3_4420_0004)); + c.cache_op(C_R10K_ISD | CACH_PI, va | 1, 0x3_4420_0004 ^ 1 << bit); + assert_eq!(c.fetch(va, pa).status, EXEC_CACHE_ERR, "bit {bit}"); + } + } + + /// A readback stored back verbatim round-trips and executes: for a + /// sampled word and for one the model encodes raw (an unsampled opcode). + #[test] + fn index_load_data_then_store_data_round_trips_and_executes() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_i_tag(&c, va, pa, 1, true); + stage_i_tag(&c, va + 0x4000, pa + 0x4000, 0, true); + for (i, (p, want)) in [(0x3_4420_0004u64, 0x2404_0002u32), (L1I_RAW << 32 | 0x7c00_0001, 0x7c00_0001)].into_iter().enumerate() { + let src = va + 4 * i as u64; + stage_p(&c, src, 1, p); + let got = c.cache_op(C_R10K_ILD | CACH_PI, src | 1, 0); + assert_eq!(got, p, "Index Load Data returns the 36 bits"); + let ecc = c.cache_op_ecc(); + assert_eq!(ecc & 1, l1i_parity_of(p)); + // Store the readback into another line, with the readback's parity. + let dst = va + 0x4000 + 4 * i as u64; + c.set_cache_ecc(ecc); + c.cache_op(C_R10K_ISD | CACH_PI, dst, got); + assert_eq!(fetch_raw(&c, dst, pa + 0x4000 + 4 * i as u64), want, "word {i}"); + } + } + + /// A staged word that differs from memory executes the staged word: a + /// stale entry stays until it is invalidated. + #[test] + fn a_staged_word_differing_from_memory_executes() { + let (c, bus) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee); + stage_i_tag(&c, va, pa, 1, true); + stage_p(&c, va, 1, 0x3_4420_0004); + // The word after it too: opcode fusion peeks at it. + stage_i_word(&c, va + 4, 1, 0); + let reads = bus.reads.load(Ordering::Relaxed); + for _ in 0..3 { + assert_eq!(fetch_raw(&c, va, pa), 0x2404_0002); + } + assert_eq!(bus.reads.load(Ordering::Relaxed), reads); + } + + /// A fetch that misses the primary I cache and hits a staged secondary + /// line refills a 64-byte primary block into an Invalid way: way 1 here, + /// with way 0 valid under an unrelated tag. Index Load Tag/Data (I) return + /// the refill, and a readback stored verbatim into another line executes. + #[test] + fn a_staged_secondary_hit_refills_the_primary_instruction_cache() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0040u64, 0xffff_ffff_8030_0040u64); + stage_tag(&c, pa & !127, 0, 2); + for q in 0..8 { + stage_data(&c, (pa & !127) + 16 * q, 0, 0x1111_0000_2222_0000 + q); + } + // Way 1 invalid, way 0 valid with an unrelated tag. + stage_i_tag(&c, va, pa, 1, false); + stage_i_tag(&c, va, pa + (1 << 20), 0, true); + // The fetch is served and refills way 1. + assert_eq!(fetch_raw(&c, va, pa), 0x2222_0004, "quadword 4"); + let tag = c.cache_op(C_ILT | CACH_PI, va | 1, 0); + assert_eq!(tag, l1i_tag_of(pa) | L1I_VALID, "way 1 holds the fetch's tag, valid"); + let way0 = c.cache_op(C_ILT | CACH_PI, va, 0); + assert_eq!(way0, l1i_tag_of(pa + (1 << 20)) | L1I_VALID, "way 0 is left alone"); + // The block is the secondary's data: the word at +0 is TagLo of the + // doubleword at +0 (the inferred order), +4 TagHi, and so on. + let dst = va + 0x4000; + stage_i_tag(&c, dst, pa + 0x4000, 0, true); + for i in 0..16u64 { + let got = c.cache_op(C_R10K_ILD | CACH_PI, (va & !63 | 4 * i) | 1, 0); + let ecc = c.cache_op_ecc(); + assert_eq!(got & 0xffff_ffff, fetch_raw(&c, (va & !63) + 4 * i, (pa & !63) + 4 * i) as u64, "word {i}"); + // Verbatim into the other line, then execute from it. + c.set_cache_ecc(ecc); + c.cache_op(C_R10K_ISD | CACH_PI, (dst & !63) + 4 * i, got); + assert_eq!(fetch_raw(&c, (dst & !63) + 4 * i, (pa & !63) + 0x4000 + 4 * i), got as u32, "stored word {i}"); + } + // The line is not refilled again once it is resident, and ending the + // secondary block ends it. + c.cache_op(C_IWBINV | CACH_SD, operand(pa & !127, 0), 0); + assert_eq!(c.cache_op(C_ILT | CACH_PI, va | 1, 0) & L1I_VALID, 0, "purged with its secondary block"); + } + + /// Both ways valid: the model picks way 0. A miss in the secondary cache + /// refills nothing. + #[test] + fn a_refill_needs_a_staged_secondary_hit() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0040u64, 0xffff_ffff_8030_0040u64); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee); + assert_eq!(c.cache_op(C_ILT | CACH_PI, va, 0) & L1I_VALID, 0); + stage_tag(&c, pa & !127, 0, 2); + stage_i_tag(&c, va, pa + (1 << 20), 0, true); + stage_i_tag(&c, va, pa + (2 << 20), 1, true); + fetch_raw(&c, va, pa); + assert_eq!(c.cache_op(C_ILT | CACH_PI, va, 0), l1i_tag_of(pa) | L1I_VALID); + } + + /// F1: a refused staged primary word never executes its raw bits, even + /// with a staged secondary line behind it; the secondary word runs. + #[test] + fn a_refused_primary_word_never_executes_its_raw_bits() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_tag(&c, pa & !127, 0, 2); + stage_data(&c, pa & !127, 0, 0x1111_0000_2222_0000); + stage_i_tag(&c, va, pa, 0, true); + stage_p(&c, va, 0, 0x5_0000_0004); // unknown class in way 0 + assert_eq!(fetch_raw(&c, va, pa), 0x2222_0000); + // No refill happened for it: way 1 stays invalid. + assert_eq!(c.cache_op(C_ILT | CACH_PI, va | 1, 0) & L1I_VALID, 0); + } + + /// S2: a guest-built word with TagHi = 0xf is not the model's raw form + /// and is refused like any unknown class. + #[test] + fn a_guest_staged_0xf_word_is_refused() { + let (c, bus) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_i_tag(&c, va, pa, 1, true); + let p = L1I_RAW << 32 | 0x2404_0002; + c.set_cache_ecc(l1i_parity_of(p)); + c.cache_op(C_R10K_ISD | CACH_PI, va | 1, p); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee); + assert!(bus.reads.load(Ordering::Relaxed) >= 1); + // Handed out by Index Load Data from a refilled line, it is raw. + assert_eq!(l1i_decode(p, false), Err(0x79)); + } + + /// The refusal fallback (an approximation): a refused + /// staged word is replaced by the staged secondary word if there is one, + /// else by memory's, and the class is reported once however often it is + /// fetched. Never the refused word's own bits. + #[test] + fn a_refused_word_falls_back_to_the_secondary_word_then_memory_and_is_reported_once() { + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + let refuse = |c: &R10000ShadowCache| { + stage_i_tag(c, va, pa, 0, true); + stage_p(c, va, 0, 0x5_0000_0004); // class 0x28, not in the table + }; + // No secondary line: memory's word, 0xeeee_eeee on this bus. + let (c, bus) = staged_cache(); + refuse(&c); + let before = bus.reads.load(Ordering::Relaxed); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee, "memory"); + assert!(bus.reads.load(Ordering::Relaxed) > before, "from the bus"); + // A staged secondary line is before memory. + let (c, _) = staged_cache(); + stage_tag(&c, pa, 0, 2); + stage_data(&c, pa, 0, 0x1111_0000_2222_0000); + refuse(&c); + assert_eq!(fetch_raw(&c, va, pa), 0x2222_0000, "the secondary word, not memory's"); + // Reported once per class, not per fetch; another class is another report. + for _ in 0..3 { + fetch_raw(&c, va, pa); + } + assert_eq!(unsafe { (*c.l1i_refused.get()).clone() }, vec![0x28], "one report for the class"); + stage_p(&c, va + 4, 0, 0x6_0000_0004); // class 0x30 + fetch_raw(&c, va + 4, pa + 4); + assert_eq!(unsafe { (*c.l1i_refused.get()).clone() }, vec![0x28, 0x30]); + } + + /// S3: a refill that evicts a guest-staged line (the last one in its page) + /// during a fetch is safe, and the evicted page goes back to the window. + #[test] + fn a_refill_may_evict_a_staged_line_during_a_fetch() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0040u64, 0xffff_ffff_8030_0040u64); + let (opa, ova) = (0x0050_0040u64, 0xffff_ffff_8050_0040u64); + stage_tag(&c, pa & !127, 0, 2); + stage_data(&c, pa & !127, 0, 0x1111_0000_2222_0000); + // Both ways of the set valid: the other page's line is the victim (way 0). + stage_i_tag(&c, va, opa, 0, true); + stage_i_word(&c, va, 0, 0x1234); + stage_i_tag(&c, va, opa + (1 << 20), 1, true); + assert!(c.page_staged(opa)); + fetch_raw(&c, va, pa); + assert!(!c.page_staged(opa), "the victim's page left the staged set"); + assert_eq!(fetch_raw(&c, ova, opa), 0xeeee_eeee, "and reads memory"); + } + + /// S5: ending a secondary block invalidates the primary I lines inside it + /// (NEC U12754EJ1V0UMJ1 pp.17, 19) and only those. An Index Store Tag (S) + /// or a hit invalidate that misses does not purge. + #[test] + fn invalidating_a_secondary_block_purges_the_primary_lines_inside_it() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + let inside = |c: &R10000ShadowCache| c.cache_op(C_ILT | CACH_PI, (va + 64) | 1, 0) & L1I_VALID != 0; + let outside = |c: &R10000ShadowCache| c.cache_op(C_ILT | CACH_PI, (va + 0x2000 + 128) | 1, 0) & L1I_VALID != 0; + let setup = || { + let (c, _) = staged_cache(); + stage_tag(&c, pa, 0, 2); + stage_i_tag(&c, va + 64, pa + 64, 1, true); // inside the 128-byte block + stage_i_tag(&c, va + 0x2000 + 128, pa + 128, 1, true); // the next block + c + }; + let c1 = setup(); + assert!(inside(&c1) && outside(&c1)); + c1.cache_op(C_IWBINV | CACH_SD, operand(pa, 0), 0); + assert!(!inside(&c1) && outside(&c1), "inside purged, outside kept"); + let c2 = setup(); + c2.cache_op(C_HINV | CACH_SD, operand(pa, 0), pa); + assert!(!inside(&c2) && outside(&c2), "Hit Invalidate hit"); + let c3 = setup(); + c3.cache_op(C_HINV | CACH_SD, operand(pa, 0), pa + (1 << 20)); // misses + stage_tag(&c3, pa, 0, 0); // Index Store Tag does not purge + assert!(inside(&c3) && outside(&c3)); + let _ = c; + } + + /// Index Store Tag (D): the primary data line of `pa` at the set `va` + /// indexes, in way 0, with `state` in TagLo[7:6] and TagHi's state + /// modifier `modifier`. + fn stage_d_tag(c: &R10000ShadowCache, va: u64, pa: u64, state: u64, modifier: u64) { + c.cache_op(C_IST | CACH_PD, va & !31, l1i_tag_of(pa) | state << 6 | modifier << 61); + } + + fn d_state(c: &R10000ShadowCache, va: u64) -> u64 { + c.cache_op(C_ILT | CACH_PD, va & !31, 0) >> 6 & 3 + } + + /// Check 10: a secondary invalidate locates the primary lines through the + /// virtual index the tag records (TagLo[8:7]), and purges the primary I + /// and D lines of the block that sit at that index. A line of the same + /// block at another index may survive. Each operation, both halves of the + /// block, and the adjacent block untouched. + #[test] + fn a_secondary_invalidate_purges_the_primary_lines_at_the_recorded_index() { + let (pa, kseg) = (0x0030_0000u64, 0xffff_ffff_8000_0000u64); + // The block's I halves and D quarters at index 1, one I half and one D + // quarter of the same block at index 0, and the next block at index 1. + let at = |idx: u64, pa: u64| kseg | idx << 12 | (pa & 0xfff); + let setup = || { + let (c, bus) = staged_cache(); + c.cache_op(C_IST | CACH_SD, operand(pa, 0), l2_tag_of(pa) | 3 << L2_STATE_SHIFT | 1 << 7); + for half in [0, 64] { + stage_i_tag(&c, at(1, pa + half), pa + half, 0, true); + } + for q in [0, 32, 64, 96] { + stage_d_tag(&c, at(1, pa + q), pa + q, 2, 1); + } + stage_i_tag(&c, at(0, pa), pa, 1, true); // same block, index 0 + stage_d_tag(&c, at(0, pa), pa, 2, 1); + stage_i_tag(&c, at(1, pa + 128), pa + 128, 0, true); // next block + stage_d_tag(&c, at(1, pa + 128), pa + 128, 2, 1); + (c, bus) + }; + let i_valid = |c: &R10000ShadowCache, idx: u64, off: u64, way: u64| + c.cache_op(C_ILT | CACH_PI, at(idx, pa + off) | way, 0) & L1I_VALID != 0; + let ops: [(&str, Box); 5] = [ + ("Index Writeback Invalidate", Box::new(move |c| { c.cache_op(C_IWBINV | CACH_SD, operand(pa, 0), 0); })), + ("Hit Invalidate, first half", Box::new(move |c| { c.cache_op(C_HINV | CACH_SD, operand(pa, 0), pa); })), + ("Hit Invalidate, second half", Box::new(move |c| { c.cache_op(C_HINV | CACH_SD, operand(pa, 0), pa + 127); })), + ("Hit Writeback Invalidate, first half", Box::new(move |c| { c.cache_op(C_HWBINV | CACH_SD, operand(pa, 0), pa); })), + ("Hit Writeback Invalidate, second half", Box::new(move |c| { c.cache_op(C_HWBINV | CACH_SD, operand(pa, 0), pa + 100); })), + ]; + for (name, op) in ops { + let (c, _) = setup(); + assert!(i_valid(&c, 1, 0, 0) && i_valid(&c, 1, 64, 0) && d_state(&c, at(1, pa)) == 2, "{name}: staged"); + op(&c); + for half in [0, 64] { + assert!(!i_valid(&c, 1, half, 0), "{name}: I half {half} at the recorded index"); + } + for q in [0, 32, 64, 96] { + assert_eq!(d_state(&c, at(1, pa + q)), 0, "{name}: D quarter {q} at the recorded index"); + } + assert!(i_valid(&c, 0, 0, 1), "{name}: I line at another index survives"); + assert_eq!(d_state(&c, at(0, pa)), 2, "{name}: D line at another index survives"); + assert!(i_valid(&c, 1, 128, 0), "{name}: next block's I line"); + assert_eq!(d_state(&c, at(1, pa + 128)), 2, "{name}: next block's D line"); + } + // The stored tag decides, not the operand: a line staged at index 1 + // survives an invalidate whose tag records index 2. + let (c, _) = staged_cache(); + c.cache_op(C_IST | CACH_SD, operand(pa, 0), l2_tag_of(pa) | 2 << L2_STATE_SHIFT | 2 << 7); + stage_i_tag(&c, at(1, pa), pa, 0, true); + stage_i_tag(&c, at(2, pa + 64), pa + 64, 0, true); + stage_d_tag(&c, at(1, pa), pa, 2, 1); + stage_d_tag(&c, at(2, pa), pa, 2, 1); + c.cache_op(C_IWBINV | CACH_SD, operand(pa, 0), 0); + assert!(i_valid(&c, 1, 0, 0) && !i_valid(&c, 2, 64, 0)); + assert_eq!((d_state(&c, at(1, pa)), d_state(&c, at(2, pa))), (2, 0)); + } + + /// A model-refilled primary I line is purged with its secondary block + /// whatever the fetch's VA[13:12] was against the tag's recorded index. + #[test] + fn a_refilled_line_is_purged_whatever_its_index() { + for tag_index in 0..4u64 { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_1000u64); // fetch index 1 + c.cache_op(C_IST | CACH_SD, operand(pa, 0), l2_tag_of(pa) | 2 << L2_STATE_SHIFT | tag_index << 7); + stage_data(&c, pa, 0, 0x1111_0000_2222_0000); + assert_eq!(fetch_raw(&c, va, pa), 0x2222_0000); + assert_ne!(c.cache_op(C_ILT | CACH_PI, va, 0) & L1I_VALID, 0, "refilled"); + c.cache_op(C_HINV | CACH_SD, operand(pa, 0), pa); + assert_eq!(c.cache_op(C_ILT | CACH_PI, va, 0) & L1I_VALID, 0, "tag index {tag_index}"); + } + } + + /// A guest that stores over a refilled line's first word makes the line + /// its own: it follows the tag's recorded index like any staged line, so a + /// Hit Invalidate (S) whose tag records another index leaves it. + #[test] + fn a_refilled_line_the_guest_stored_over_follows_the_recorded_index() { + for (tag_index, survives) in [(1u64, false), (2, true)] { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_1000u64); // fetch index 1 + c.cache_op(C_IST | CACH_SD, operand(pa, 0), l2_tag_of(pa) | 2 << L2_STATE_SHIFT | tag_index << 7); + stage_data(&c, pa, 0, 0x1111_0000_2222_0000); + assert_eq!(fetch_raw(&c, va, pa), 0x2222_0000); + stage_p(&c, va, 0, 0x3_4420_0004); // predecoded, not the raw form: the flag goes + c.cache_op(C_HINV | CACH_SD, operand(pa, 0), pa); + let valid = c.cache_op(C_ILT | CACH_PI, va, 0) & L1I_VALID != 0; + assert_eq!(valid, survives, "tag index {tag_index}"); + } + } + + /// Check 10, writeback: Index and Hit Writeback Invalidate (S) write a + /// written-and-inconsistent primary D line (modifier 010) back to the + /// secondary line before purging it, so a DirtyExclusive block carries + /// it to the system interface; Hit Invalidate (S) writes nothing. A + /// consistent line (modifier 001) carries nothing. + #[test] + fn an_inconsistent_data_line_is_written_back_before_it_is_purged() { + let (pa, kseg) = (0x0030_0000u64, 0xffff_ffff_8000_0000u64); + let va = kseg | (pa + 32) & 0xfff; + for (name, hit_op, writes, modifier) in [ + ("Index Writeback Invalidate", C_IWBINV, true, 2), + ("Hit Writeback Invalidate", C_HWBINV, true, 2), + ("Hit Invalidate", C_HINV, false, 2), + ("Index Writeback Invalidate, consistent", C_IWBINV, true, 1), + ] { + let (c, bus) = staged_cache(); + c.cache_op(C_IST | CACH_SD, operand(pa, 0), l2_tag_of(pa) | 3 << L2_STATE_SHIFT); + stage_d_tag(&c, va, pa + 32, 3, modifier); + // Words 0 and 1 of the line's first doubleword; the others are zero. + c.cache_op(C_R10K_ISD | CACH_PD, va & !3, 0x1111_1111); + c.cache_op(C_R10K_ISD | CACH_PD, (va & !3) + 4, 0x2222_2222); + if hit_op == C_IWBINV { + c.cache_op(C_IWBINV | CACH_SD, operand(pa, 0), 0); + } else { + c.cache_op(hit_op | CACH_SD, operand(pa, 0), pa); + } + let w = bus.writes.lock().unwrap().clone(); + assert_eq!(d_state(&c, va), 0, "{name}: purged"); + if !writes { + assert!(w.is_empty(), "{name}: nothing reaches memory"); + } else if modifier == 2 { + // The block's 16 doublewords go out; the D line is the fifth to eighth. + assert_eq!(w.len(), 16, "{name}"); + assert_eq!(w[4], (pa + 32, 0x1111_1111_2222_2222), "{name}: the doubleword with the D line's words"); + assert!(w.iter().enumerate().all(|(i, &(_, v))| i == 4 || v == 0), "{name}: the rest"); + } else { + assert!(w.iter().all(|&(_, v)| v == 0), "{name}: a consistent line carries nothing"); + } + } + } + + /// A page the decode cache has already read through ppmem's window must go + /// word by word once a primary I line is staged in it, or the cached slot + /// keeps serving the window's word (`serve_window` drops the window), and + /// must go back to the window when the line ends. The other staged tests + /// have no window, so the switch is invisible to them. + #[test] + fn a_page_decoded_from_the_window_is_read_by_word_once_a_line_is_staged() { + let (c, bus) = staged_cache(); + let mut window = vec![0u32; 0x4000]; // 64 KiB of RAM: pages 0 to 15 + let (pa, va) = (0x0000_3040u64, 0xffff_ffff_8000_3040u64); + window[(pa >> 2) as usize] = 0x2402_0001; + unsafe { c.set_tcache_window(window.as_mut_ptr() as *mut u8) }; + unsafe { *c.tcache_bitmap_ptr() = 1 }; + assert_eq!(fetch_raw(&c, va, pa), 0x2402_0001, "decoded from the window"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "the bus was not read"); + stage_i_tag(&c, va, pa, 0, true); + stage_i_word(&c, va, 0, 0x2402_0002); + assert_eq!(fetch_raw(&c, va, pa), 0x2402_0002, "the staged word, not the cached slot"); + c.cache_op(C_IINV | CACH_PI, va, 0); + assert_eq!(fetch_raw(&c, va, pa), 0x2402_0001, "the window again once the line ends"); + } + + /// Check 11, the cases the earlier test leaves out. The victim of a + /// refill is an Invalid way before a Valid one: way 0 Invalid with way 1 + /// Valid fills way 0 and leaves way 1; both Invalid fills way 0 (a model + /// choice, as is both Valid). Each case reads the whole set back. + #[test] + fn the_refill_victim_follows_the_ways_that_are_invalid() { + let (pa, va) = (0x0030_0040u64, 0xffff_ffff_8030_0040u64); + let (other0, other1) = (pa + (1 << 20), pa + (2 << 20)); + // (way 0 valid, way 1 valid, the way the refill fills) + for (v0, v1, fills) in [(false, true, 0u64), (true, false, 1), (false, false, 0), (true, true, 0)] { + let (c, _) = staged_cache(); + stage_tag(&c, pa & !127, 0, 2); + stage_data(&c, pa, 0, 0x1111_0000_2222_0000); + stage_i_tag(&c, va, other0, 0, v0); + stage_i_tag(&c, va, other1, 1, v1); + let before = [c.cache_op(C_ILT | CACH_PI, va, 0), c.cache_op(C_ILT | CACH_PI, va | 1, 0)]; + assert_eq!(fetch_raw(&c, va, pa), 0x2222_0000); + let after = [c.cache_op(C_ILT | CACH_PI, va, 0), c.cache_op(C_ILT | CACH_PI, va | 1, 0)]; + for way in 0..2 { + if way == fills { + assert_eq!(after[way as usize], l1i_tag_of(pa) | L1I_VALID, + "way 0 valid {v0}, way 1 valid {v1}: way {way} holds the block"); + } else { + assert_eq!(after[way as usize], before[way as usize], + "way 0 valid {v0}, way 1 valid {v1}: way {way} untouched"); + } + } + } + } + + /// The primary I tag holds PA[39:36] in TagHi[3:0] and a fetch compares + /// it: a line staged for a block whose PA differs only in one of those + /// bits does not hit this block, and hits its own. + #[test] + fn a_primary_instruction_line_compares_physical_address_bits_39_to_36() { + for bit in 36..40 { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_i_tag(&c, va, pa | 1 << bit, 0, true); + stage_i_word(&c, va, 0, 0x1234); + assert_eq!(fetch_raw(&c, va, pa), 0xeeee_eeee, "PA bit {bit} differs: a miss"); + assert_eq!(fetch_raw(&c, va, pa | 1 << bit), 0x1234, "PA bit {bit} matches: a hit"); + } + } + + /// The set of a secondary line is PA[N-2:B]. The geometry test's lines all + /// have the top index bit clear, which a lookup fixed at 4096 sets also + /// passes. Set the top bit (and the one below it) at every cache size and + /// find, load, hit-invalidate and index-invalidate the line. + #[test] + fn the_top_secondary_index_bit_reaches_the_line_at_every_cache_size() { + fn check>>(name: &str) { + let line = C::L2_LINE as u64; + let way_bytes = (C::L2_SIZE / 2) as u64; + let top = way_bytes >> 1; + for a in [0x0020_3000u64 + top, 0x0020_3000 + (top >> 1), 0x0020_3000 + top + (top >> 1)] { + let bus = Arc::new(Bus::default()); + let c = C::from(bus.clone() as Arc); + let op = |pa: u64, way: u64| 0xffff_ffff_8000_0000 | (pa & 0x1fff_fff8) | way; + let tag = |pa: u64, way: u64, state: u64| { + c.cache_op(C_IST | CACH_SD, op(pa & !(line - 1), way), l2_tag_of(pa) | state << L2_STATE_SHIFT); + }; + let load = |pa: u64| c.read::<8>(op(pa, 0), pa).data; + tag(a, 1, 2); + c.cache_op(C_R10K_ISD | CACH_SD, op(a & !7, 1), 0x0a0a_0a0a_0b0b_0b0b); + assert_eq!(load(a), 0x0b0b_0b0b_0a0a_0a0a, "{name}: {a:#x} hits"); + assert_eq!(bus.reads.load(Ordering::Relaxed), 0, "{name}: {a:#x}"); + // The line of the same set without the top bit is another set. + assert_eq!(load(a ^ top), MEM, "{name}: {a:#x} with the top bit flipped"); + c.cache_op(C_HINV | CACH_SD, op(a, 0), a); + assert_eq!(load(a), MEM, "{name}: {a:#x} Hit Invalidate ends it"); + tag(a, 1, 2); + c.cache_op(C_IWBINV | CACH_SD, op(a, 1), 0); + assert_eq!(load(a), MEM, "{name}: {a:#x} Index Writeback Invalidate ends it"); + } + } + check::("R10000 1 MiB/128"); + check::("R10000 O2 1 MiB/64"); + check::("R12000 2 MiB/128"); + } + + /// The set of raw words Index Load Data handed out is bounded: after + /// `L1I_ISSUED_MAX` it starts again, and the newest word is always in it. + #[test] + fn the_issued_raw_words_are_bounded() { + let (c, _) = staged_cache(); + for i in 0..(L1I_ISSUED_MAX as u64 * 2 + 5) { + c.l1i_note_issued(L1I_RAW << 32 | i); + let issued = unsafe { &*c.l1i_issued.get() }; + assert!(issued.len() <= L1I_ISSUED_MAX, "word {i}"); + assert!(issued.contains(&(L1I_RAW << 32 | i))); + } + } + + /// S1: parity suppression follows the real Status register (DE, ERL), not + /// a direct call, and does not need the cache-ecc feature. + #[test] + fn status_de_and_erl_suppress_the_parity_error() { + big_stack(|| { + use crate::cpu::mips_core::{STATUS_DE, STATUS_ERL}; + let (mut exec, _) = mapped(3, 1, 0x3a5); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + let p = 0x3_4420_0004u64; + cache_at(&mut exec, 0x08, va | 1, l1i_tag_of(pa) | L1I_VALID); + exec.core.cp0_ecc = l1i_parity_of(p) ^ 1; + cache_at(&mut exec, 0x1c, va | 1, p); + exec.core.pc = va; + let status = exec.core.cp0_status & !(STATUS_DE | STATUS_ERL); + exec.set_cp0_status(status); + assert_eq!(exec.cache.fetch(va, pa).status, EXEC_CACHE_ERR, "detected"); + exec.set_cp0_status(status | STATUS_DE); + assert_eq!(exec.cache.fetch(va, pa).status, crate::cpu::mips_exec::EXEC_COMPLETE, "DE set"); + exec.set_cp0_status(status | STATUS_ERL); + assert_eq!(exec.cache.fetch(va, pa).status, crate::cpu::mips_exec::EXEC_COMPLETE, "ERL set"); + exec.set_cp0_status(status); + assert_eq!(exec.cache.fetch(va, pa).status, EXEC_CACHE_ERR, "detected again"); + }); + } + + /// A snapshot that fails to parse leaves the staged lines as they were. + #[test] + fn a_failed_restore_changes_nothing() { + let (c, _) = staged_cache(); + stage_tag(&c, PA, 0, 2); + stage_data(&c, PA, 0, 0x1234); + let mut bad = c.save_cache_state(); + bad.as_table_mut().unwrap().insert("staged_l1i".into(), toml::Value::Array(vec![toml::Value::Table(Default::default())])); + assert!(c.load_cache_state(&bad).is_err()); + assert_ne!(load::<8>(&c, PA), MEM, "the staged line survived"); + } + + /// Staged primary I lines survive a snapshot round trip. + #[test] + fn staged_primary_instruction_lines_are_saved() { + let (c, _) = staged_cache(); + let (pa, va) = (0x0030_0000u64, 0xffff_ffff_8030_0000u64); + stage_i_tag(&c, va, pa, 1, true); + for w in 0..16 { + stage_i_word(&c, va + 4 * w, 1, 0x7000 + w as u32); + } + let saved = c.save_cache_state(); + let (d, _) = staged_cache(); + d.load_cache_state(&saved).unwrap(); + assert_eq!(fetch_raw(&d, va + 12, pa + 12), 0x7003); + assert!(d.page_staged(pa)); + d.load_cache_state(&toml::Value::Table(Default::default())).unwrap(); + assert_eq!(fetch_raw(&d, va + 12, pa + 12), 0xeeee_eeee); + } + // ---- Mapped operands ------------------------------------------------ type MappedExec = crate::cpu::mips_exec::MipsExecutor; diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 5b7ef9130..54add462e 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -859,6 +859,14 @@ pub trait MipsCache: Send + Sync { /// are detected at all (neither DE nor ERL set). Pushed by the executor on /// every Status change. Models that keep no check bits ignore it. fn set_error_control(&self, _ce: bool, _detect: bool) {} + /// Where to read the CPU's PC and Status registers, for a model that + /// needs them only rarely instead of being told of every change + /// (`set_error_control`): the R10000 shadow cache's staged lines. Handed + /// over by the executor on the CPU thread, with the executor in place. + /// + /// # Safety + /// Both must outlive the cache, which reads them on the CPU thread. + unsafe fn set_cpu_registers(&self, _pc: *const u64, _status: *const u32) {} /// CacheErr for the Cache Error exception the last access reported /// (`EXEC_CACHE_ERR`). fn take_cache_err(&self) -> u32 { 0 } diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 5977dcb30..b45ab5466 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -3370,6 +3370,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[cold] #[inline(never)] pub(crate) fn staged_lines_changed(&mut self) { + // A fetch from a staged line reads the PC, and its parity check DE and + // ERL. Handed over here, on the CPU thread with the executor in place. + unsafe { self.cache.set_cpu_registers(&self.core.pc, &self.core.cp0_status) }; self.nanotlb_invalidate(); #[cfg(feature = "jitv2")] {