From 4146e3f159524846f2715a1348dd6fceb5f55c20 Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Tue, 18 Aug 2026 20:34:37 +0000 Subject: [PATCH] harden AArch64 gate signal recovery --- .../src/aarch64.rs | 381 +++++++++++----- litebox_syscall_rewriter/src/aarch64.rs | 405 ++++++++++++++---- 2 files changed, 607 insertions(+), 179 deletions(-) diff --git a/litebox_platform_linux_userland/src/aarch64.rs b/litebox_platform_linux_userland/src/aarch64.rs index 0a54ec450..09bc70549 100644 --- a/litebox_platform_linux_userland/src/aarch64.rs +++ b/litebox_platform_linux_userland/src/aarch64.rs @@ -7,7 +7,9 @@ use super::*; use litebox_syscall_rewriter::aarch64::{ GATE_ALIGNMENT, GATE_PC_CANDIDATE_COUNT, GATE_SLOT_SIZES, GateMetadata, MSR_FRAME_BYTES, - SVC_FRAME_BYTES, SVC_GATE_BYTES, SVC_SLOT_BYTES, + MSR_FRAME_OFF_VALUE, MrsTpidrGateOffset, MrsTpidrValueSource, MsrTpidrFrameState, + MsrTpidrGateOffset, RuntimeAccess, SVC_FRAME_BYTES, SVC_GATE_BYTES, SVC_SLOT_BYTES, + SvcFrameState, SvcGateOffset, }; #[cfg(feature = "aarch64_virtualize_x18")] use litebox_syscall_rewriter::aarch64::{ @@ -1177,6 +1179,13 @@ pub(super) enum GateInterruption { Asynchronous, } +#[derive(Clone, Copy)] +struct GateRuntimeState { + guest_thread_pointer_addr: usize, + expected_outbound_stub: usize, + expected_outbound_pc: usize, +} + fn read_usize(read: &mut impl FnMut(usize, &mut [u8]) -> bool, address: usize) -> Option { let mut bytes = [0u8; size_of::()]; read(address, &mut bytes).then(|| usize::from_ne_bytes(bytes)) @@ -1193,7 +1202,7 @@ fn recover_x18_frame( X18FrameState::AtSpRegistersLive => (signal_sp, false, true), X18FrameState::AtSpRestoreRegisters => (signal_sp, true, true), X18FrameState::BelowSpRestoreRegisters => { - let frame_sp = signal_sp.checked_sub(usize::from(X18_FRAME_BYTES))?; + let frame_sp = signal_sp.wrapping_sub(usize::from(X18_FRAME_BYTES)); (frame_sp, true, false) } }; @@ -1207,7 +1216,7 @@ fn recover_x18_frame( None }; let final_sp = if pop_frame { - frame_sp.checked_add(usize::from(X18_FRAME_BYTES))? + frame_sp.wrapping_add(usize::from(X18_FRAME_BYTES)) } else { signal_sp }; @@ -1255,9 +1264,11 @@ pub(super) fn canonicalize_aarch64_gate_signal_context( canonicalize_aarch64_gate_signal_context_with_kind( context, saved, - guest_thread_pointer_addr, - expected_outbound_stub, - expected_outbound_pc, + GateRuntimeState { + guest_thread_pointer_addr, + expected_outbound_stub, + expected_outbound_pc, + }, GateInterruption::Asynchronous, read, ) @@ -1266,9 +1277,7 @@ pub(super) fn canonicalize_aarch64_gate_signal_context( fn canonicalize_aarch64_gate_signal_context_with_kind( context: &libc::ucontext_t, saved: &litebox_common_linux::PtRegs, - guest_thread_pointer_addr: usize, - expected_outbound_stub: usize, - expected_outbound_pc: usize, + runtime: GateRuntimeState, interruption: GateInterruption, mut read: impl FnMut(usize, &mut [u8]) -> bool, ) -> Aarch64GateSignalResult { @@ -1276,9 +1285,6 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( // boundaries below follow each emitted template. const SVC_FRAME: usize = SVC_FRAME_BYTES as usize; const MSR_FRAME: usize = MSR_FRAME_BYTES as usize; - #[cfg(not(feature = "aarch64_virtualize_x18"))] - let _ = interruption; - let pc = context.uc_mcontext.pc; if !pc.is_multiple_of(4) { return Aarch64GateSignalResult::NotGate; @@ -1361,8 +1367,8 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( let metadata = gate.metadata(); let offset = pc - slot_start; if matches!(metadata, GateMetadata::Svc) && offset >= SVC_GATE_BYTES { - return if expected_outbound_stub == slot_start + SVC_GATE_BYTES - && expected_outbound_pc == site + 4 + return if runtime.expected_outbound_stub == slot_start + SVC_GATE_BYTES + && runtime.expected_outbound_pc == site + 4 { Aarch64GateSignalResult::PreserveSavedContext } else { @@ -1381,35 +1387,73 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( match metadata { GateMetadata::MrsTpidr { destination, .. } => { let destination = usize::from(destination); - if offset == 4 { - let Some(guest_tp) = read_usize(&mut read, guest_thread_pointer_addr) else { - return Aarch64GateSignalResult::InvalidRuntimeState; - }; - canonical.regs[destination] = guest_tp; - canonical.pc = site + 4; - } else if offset == 8 { + let Some(stage) = MrsTpidrGateOffset::from_offset(offset) else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + let Some(recovery) = stage.recovery_plan() else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + if recovery.runtime_access == RuntimeAccess::Memory + && interruption == GateInterruption::Synchronous + { + return Aarch64GateSignalResult::InvalidRuntimeState; + } + match recovery.value { + MrsTpidrValueSource::Register => {} + MrsTpidrValueSource::Slot => { + let Some(guest_tp) = read_usize(&mut read, runtime.guest_thread_pointer_addr) + else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + canonical.regs[destination] = guest_tp; + } + } + if recovery.completed { canonical.pc = site + 4; } } GateMetadata::MsrTpidr { source, .. } => { - if offset == 4 { - let Some(sp) = canonical.sp.checked_add(MSR_FRAME) else { - return Aarch64GateSignalResult::NotGate; - }; - canonical.sp = sp; - } else if (8..28).contains(&offset) { + let Some(stage) = MsrTpidrGateOffset::from_offset(offset) else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + let Some(recovery) = stage.recovery_plan() else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + if recovery.runtime_access == RuntimeAccess::Memory + && interruption == GateInterruption::Synchronous + { + return Aarch64GateSignalResult::InvalidRuntimeState; + } + let restored = if recovery.frame == MsrTpidrFrameState::RestoreRegisters { let mut frame = [[0u8; size_of::()]; 3]; - let words = if offset == 8 { 2 } else { 3 }; - if !read(canonical.sp, frame[..words].as_flattened_mut()) { + if !read(canonical.sp, frame.as_flattened_mut()) { return Aarch64GateSignalResult::NotGate; } let saved_x16 = usize::from_ne_bytes(frame[0]); let saved_x17 = usize::from_ne_bytes(frame[1]); - if offset >= 12 { - let captured = usize::from_ne_bytes(frame[2]); + let captured = usize::from_ne_bytes(frame[2]); + let architectural_source = match source { + 16 => saved_x16, + 17 => saved_x17, + 31 => 0, + register => canonical.regs[usize::from(register)], + }; + if captured != architectural_source { + return Aarch64GateSignalResult::NotGate; + } + Some([saved_x16, saved_x17]) + } else { + // Before commit, the staged value cheaply checks consistency + // with the interrupted frame. After LDP at +28, recovery needs + // neither the frame nor this check. + if recovery.validate_capture { + let Some(captured) = read_usize( + &mut read, + canonical.sp.wrapping_add(usize::from(MSR_FRAME_OFF_VALUE)), + ) else { + return Aarch64GateSignalResult::NotGate; + }; let architectural_source = match source { - 16 => saved_x16, - 17 => saved_x17, 31 => 0, register => canonical.regs[usize::from(register)], }; @@ -1417,40 +1461,56 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( return Aarch64GateSignalResult::NotGate; } } + None + }; + let guest_sp = match recovery.frame { + MsrTpidrFrameState::Absent => canonical.sp, + MsrTpidrFrameState::RegistersLive | MsrTpidrFrameState::RestoreRegisters => { + canonical.sp.wrapping_add(MSR_FRAME) + } + }; + if let Some([saved_x16, saved_x17]) = restored { canonical.regs[16] = saved_x16; canonical.regs[17] = saved_x17; - let Some(sp) = canonical.sp.checked_add(MSR_FRAME) else { - return Aarch64GateSignalResult::NotGate; - }; - canonical.sp = sp; - } else if offset == 28 { - // LDP at +24 has completed, so x16/x17 are already the live - // guest values. ADD SP at +28 has not yet completed. - let Some(sp) = canonical.sp.checked_add(MSR_FRAME) else { - return Aarch64GateSignalResult::NotGate; - }; - canonical.sp = sp; } - if offset > usize::from(gate.commit_offset()) { + canonical.sp = guest_sp; + if recovery.completed { canonical.pc = site + 4; } } GateMetadata::Svc => { - if offset == 4 { - let Some(sp) = canonical.sp.checked_add(SVC_FRAME) else { - return Aarch64GateSignalResult::NotGate; - }; - canonical.sp = sp; - } else if offset >= 8 { - let Some(saved_x16) = read_usize(&mut read, canonical.sp) else { - return Aarch64GateSignalResult::NotGate; - }; + let Some(stage) = SvcGateOffset::from_offset(offset) else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + let Some(plan) = stage.recovery_plan() else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; + // SVC uses a template-fixed literal; supported MRS/MSR execution + // uses the permanent host anchor read by the gate itself. + if plan.runtime_access == RuntimeAccess::Memory + && interruption == GateInterruption::Synchronous + { + return Aarch64GateSignalResult::InvalidRuntimeState; + } + let saved_x16 = match plan.frame { + SvcFrameState::NoFrame | SvcFrameState::FrameAtSpX16Live => None, + SvcFrameState::RestoreX16FromFrame => { + let Some(saved_x16) = read_usize(&mut read, canonical.sp) else { + return Aarch64GateSignalResult::NotGate; + }; + Some(saved_x16) + } + }; + let guest_sp = match plan.frame { + SvcFrameState::NoFrame => canonical.sp, + SvcFrameState::FrameAtSpX16Live | SvcFrameState::RestoreX16FromFrame => { + canonical.sp.wrapping_add(SVC_FRAME) + } + }; + if let Some(saved_x16) = saved_x16 { canonical.regs[16] = saved_x16; - let Some(sp) = canonical.sp.checked_add(SVC_FRAME) else { - return Aarch64GateSignalResult::NotGate; - }; - canonical.sp = sp; } + canonical.sp = guest_sp; } #[cfg(feature = "aarch64_virtualize_x18")] GateMetadata::X18 { scratch } => { @@ -1468,14 +1528,14 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( let Some((guest_sp, restored_registers)) = recover_x18_frame(plan.frame, canonical.sp, &mut read) else { - return Aarch64GateSignalResult::InvalidRuntimeState; + return Aarch64GateSignalResult::NotGate; }; let guest_x18 = match plan.value { X18ValueSource::Scratch => context.uc_mcontext.regs[usize::from(scratch)].trunc(), X18ValueSource::Slot => { let Some(value) = read_usize( &mut read, - guest_thread_pointer_addr + runtime.guest_thread_pointer_addr + litebox_syscall_rewriter::aarch64::GUEST_X18_OFFSET_FROM_GUEST_TP, ) else { return Aarch64GateSignalResult::InvalidRuntimeState; @@ -1498,20 +1558,22 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( let Some(stage) = X18CompareBranchOffset::from_offset(offset) else { return Aarch64GateSignalResult::InvalidRuntimeState; }; + let Some(plan) = stage.recovery_plan() else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; if interruption == GateInterruption::Synchronous - && stage.recovery_plan().slot_access != X18SlotAccess::None + && plan.slot_access != X18SlotAccess::None { return Aarch64GateSignalResult::InvalidRuntimeState; } - let plan = stage.recovery_plan(); let Some((guest_sp, restored_registers)) = recover_x18_frame(plan.frame, canonical.sp, &mut read) else { - return Aarch64GateSignalResult::InvalidRuntimeState; + return Aarch64GateSignalResult::NotGate; }; let Some(guest_x18) = read_usize( &mut read, - guest_thread_pointer_addr + runtime.guest_thread_pointer_addr + litebox_syscall_rewriter::aarch64::GUEST_X18_OFFSET_FROM_GUEST_TP, ) else { return Aarch64GateSignalResult::InvalidRuntimeState; @@ -1541,23 +1603,25 @@ fn canonicalize_aarch64_gate_signal_context_with_kind( let Some(stage) = X18AdrOffset::from_offset(offset) else { return Aarch64GateSignalResult::InvalidRuntimeState; }; + let Some(plan) = stage.recovery_plan() else { + return Aarch64GateSignalResult::InvalidRuntimeState; + }; if interruption == GateInterruption::Synchronous - && stage.recovery_plan().slot_access != X18SlotAccess::None + && plan.slot_access != X18SlotAccess::None { return Aarch64GateSignalResult::InvalidRuntimeState; } - let plan = stage.recovery_plan(); let Some((guest_sp, restored_registers)) = recover_x18_frame(plan.frame, canonical.sp, &mut read) else { - return Aarch64GateSignalResult::InvalidRuntimeState; + return Aarch64GateSignalResult::NotGate; }; let guest_x18 = match plan.value { X18ValueSource::Scratch => context.uc_mcontext.regs[usize::from(scratch)].trunc(), X18ValueSource::Slot => { let Some(value) = read_usize( &mut read, - guest_thread_pointer_addr + runtime.guest_thread_pointer_addr + litebox_syscall_rewriter::aarch64::GUEST_X18_OFFSET_FROM_GUEST_TP, ) else { return Aarch64GateSignalResult::InvalidRuntimeState; @@ -1613,9 +1677,11 @@ pub(super) fn canonicalize_runtime_aarch64_gate_signal_context( canonicalize_aarch64_gate_signal_context_with_kind( context, saved, - block + tls_offset::GUEST_THREAD_POINTER, - expected_outbound_stub, - expected_outbound_pc, + GateRuntimeState { + guest_thread_pointer_addr: block + tls_offset::GUEST_THREAD_POINTER, + expected_outbound_stub, + expected_outbound_pc, + }, interruption, |address, output| { // SAFETY: `output` is writable for its exact length. The source is @@ -2062,7 +2128,7 @@ mod tests { for offset in (0usize..=44).step_by(4) { context.uc_mcontext.pc = 0x400010 + offset as u64; context.uc_mcontext.sp = if offset == 0 { 0x8020 } else { 0x8000 }; - context.uc_mcontext.regs[16] = if offset < 8 { + context.uc_mcontext.regs[16] = if offset <= 8 { saved.regs[16] as u64 } else if (20..28).contains(&offset) { TRAMPOLINE_ADDRESS @@ -2231,6 +2297,47 @@ mod tests { ); } + for (offset, signal_sp, frame_address, expected_sp) in [ + (20usize, 8usize, 8usize.wrapping_sub(16), 8usize), + (24, 8, 8usize.wrapping_sub(16), 8), + ( + 8, + usize::MAX.wrapping_sub(15), + usize::MAX.wrapping_sub(15), + 0, + ), + ] { + let mut context: libc::ucontext_t = unsafe { core::mem::zeroed() }; + context.uc_mcontext.pc = (SLOT + offset) as u64; + context.uc_mcontext.sp = signal_sp as u64; + context.uc_mcontext.regs[16] = SAVED_ANCHOR as u64; + context.uc_mcontext.regs[17] = SAVED_SCRATCH as u64; + let mut slots = [0u8; 16]; + slots[8..].copy_from_slice(&LOGICAL_BEFORE.to_ne_bytes()); + let mut reader = fixture_reader(&code, &trampoline, &[], &slots); + let result = canonicalize_aarch64_gate_signal_context( + &context, + &saved, + 0x500000, + 0, + 0, + move |address, output| { + if address == frame_address && output.len() == frame.len() { + output.copy_from_slice(&frame); + true + } else { + reader(address, output) + } + }, + ); + let Aarch64GateSignalResult::Canonicalized(regs) = result else { + panic!("x18 +{offset} rejected wrapped frame arithmetic"); + }; + assert_eq!(regs.sp, expected_sp, "+{offset}: SP"); + assert_eq!(regs.regs[16], SAVED_ANCHOR, "+{offset}: anchor scratch"); + assert_eq!(regs.regs[17], SAVED_SCRATCH, "+{offset}: value scratch"); + } + for offset in [8usize, 32] { let mut context: libc::ucontext_t = unsafe { core::mem::zeroed() }; context.uc_mcontext.pc = (SLOT + offset) as u64; @@ -2240,9 +2347,11 @@ mod tests { let result = canonicalize_aarch64_gate_signal_context_with_kind( &context, &saved, - 0x500000, - 0, - 0, + GateRuntimeState { + guest_thread_pointer_addr: 0x500000, + expected_outbound_stub: 0, + expected_outbound_pc: 0, + }, GateInterruption::Synchronous, fixture_reader(&code, &trampoline, &frame, &[0; 16]), ); @@ -2289,10 +2398,7 @@ mod tests { 0, fixture_reader(&code, &trampoline, &[], &[0; 16]), ); - assert!(matches!( - result, - Aarch64GateSignalResult::InvalidRuntimeState - )); + assert!(matches!(result, Aarch64GateSignalResult::NotGate)); } #[test] @@ -2475,11 +2581,8 @@ mod tests { frame[16..24].copy_from_slice(&(saved.regs[5] as u64).to_le_bytes()); let before = ptregs_bytes(&saved); - // The slot body, the original site and the gate frame all live in guest - // memory, so denying any of them yields `NotGate` and leaves the signal - // with the guest; see the boundary on - // `canonicalize_aarch64_gate_signal_context`. None of them may touch - // the saved registers on the way out. + // Candidate, original-site, and guest-frame provenance failures are + // `NotGate`. No failure may mutate the saved registers. for denied_address in [0x400010usize, 0x1000, 0x8000] { let mut reader = fixture_reader(&code, &trampoline, &frame, &[]); let result = super::aarch64::canonicalize_aarch64_gate_signal_context( @@ -2492,9 +2595,13 @@ mod tests { ); assert!( matches!(result, Aarch64GateSignalResult::NotGate), - "denying {denied_address:#x} should leave the signal with the guest" + "denying {denied_address:#x} should be NotGate" + ); + assert_eq!( + ptregs_bytes(&saved), + before, + "denying {denied_address:#x} mutated saved PtRegs" ); - assert_eq!(ptregs_bytes(&saved), before); } let mut wrong_branch = code.clone(); @@ -2694,9 +2801,7 @@ mod tests { ); assert_failure(result, true, "ambiguous candidates"); - // SP restoration arithmetic must be checked before publishing a result. - // `sp` is a guest register, so a value that cannot be restored means - // this is not a gate frame rather than a broken runtime. + // Gate stack arithmetic wraps exactly like the AArch64 ADD/SUB pair. context.uc_mcontext.pc = 0x400010 + 4; context.uc_mcontext.sp = u64::MAX - 15; let result = super::aarch64::canonicalize_aarch64_gate_signal_context( @@ -2707,7 +2812,10 @@ mod tests { 0, fixture_reader(&code, &trampoline, &frame, &[]), ); - assert_failure(result, false, "SP overflow"); + let Aarch64GateSignalResult::Canonicalized(regs) = result else { + panic!("wrapped SP did not canonicalize"); + }; + assert_eq!(regs.sp, 16); } #[test] @@ -2792,12 +2900,45 @@ mod tests { 0, fixture_reader(&msr_code, &msr_slot, &[], &[]), ); - assert_eq!( - matches!(result, Aarch64GateSignalResult::Canonicalized(_)), - matches!(offset, 0 | 4 | 28 | 32), - "MSR +{offset} frame requirement" - ); + if matches!(offset, 0 | 4 | 8 | 28 | 32) { + assert!( + matches!(result, Aarch64GateSignalResult::Canonicalized(_)), + "MSR +{offset} should not need the frame" + ); + } else { + assert!( + matches!(result, Aarch64GateSignalResult::NotGate), + "MSR +{offset} should require the frame" + ); + } } + context.uc_mcontext.pc = 0x400010 + 12; + context.uc_mcontext.sp = 0x8000; + context.uc_mcontext.regs[5] = saved.regs[5] as u64; + let captured = saved.regs[5].to_ne_bytes(); + let mut reader = fixture_reader(&msr_code, &msr_slot, &[], &[]); + let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + &context, + &saved, + 0x500000, + 0, + 0, + move |address, output| { + if address == 0x8010 && output.len() == captured.len() { + output.copy_from_slice(&captured); + true + } else { + reader(address, output) + } + }, + ); + let Aarch64GateSignalResult::Canonicalized(regs) = result else { + panic!("MSR +12 required saved x16/x17"); + }; + assert_eq!(regs.regs[16], context.uc_mcontext.regs[16].trunc()); + assert_eq!(regs.regs[17], context.uc_mcontext.regs[17].trunc()); + assert_eq!(regs.sp, 0x8020); + // The frame value is provenance before and after commit; changing it // must fail rather than reconstructing from scratch registers. msr_frame[16..].copy_from_slice(&0xdead_beef_dead_beefu64.to_ne_bytes()); @@ -2830,10 +2971,7 @@ mod tests { 0, fixture_reader(&svc_code, &svc_slot, &[], &[]), ); - // Past the staging store the frame is required, and it lives on the - // guest's stack -- so an unreadable one leaves the signal with the - // guest rather than killing the runtime. - if matches!(offset, 0 | 4) { + if matches!(offset, 0 | 4 | 8) { assert!( matches!(result, Aarch64GateSignalResult::Canonicalized(_)), "SVC +{offset} should not need the frame" @@ -2841,10 +2979,53 @@ mod tests { } else { assert!( matches!(result, Aarch64GateSignalResult::NotGate), - "SVC +{offset} with an unreadable frame must stay with the guest" + "SVC +{offset} should require the frame" ); } } + + let mut context: libc::ucontext_t = unsafe { core::mem::zeroed() }; + let no_frame: &[u8] = &[]; + for (code, slot, offset, frame) in [ + (&mrs_code, &mrs_slot, 4usize, no_frame), + (&msr_code, &msr_slot, 20, msr_frame.as_slice()), + (&svc_code, &svc_slot, 28, no_frame), + ] { + context.uc_mcontext.pc = 0x400010 + offset as u64; + context.uc_mcontext.sp = 0x8000; + let result = canonicalize_aarch64_gate_signal_context_with_kind( + &context, + &saved, + GateRuntimeState { + guest_thread_pointer_addr: 0x500000, + expected_outbound_stub: 0, + expected_outbound_pc: 0, + }, + GateInterruption::Synchronous, + fixture_reader(code, slot, frame, &guest_tp), + ); + assert!( + matches!(result, Aarch64GateSignalResult::InvalidRuntimeState), + "gate +{offset} synchronous runtime access" + ); + } + + for (code, slot) in [(&svc_code, &svc_slot), (&msr_code, &msr_slot)] { + context.uc_mcontext.pc = 0x400010 + 4; + context.uc_mcontext.sp = usize::MAX.wrapping_sub(31) as u64; + let result = canonicalize_aarch64_gate_signal_context( + &context, + &saved, + 0x500000, + 0, + 0, + fixture_reader(code, slot, &[], &guest_tp), + ); + let Aarch64GateSignalResult::Canonicalized(regs) = result else { + panic!("gate failed to restore wrapped guest SP"); + }; + assert_eq!(regs.sp, 0); + } } #[test] diff --git a/litebox_syscall_rewriter/src/aarch64.rs b/litebox_syscall_rewriter/src/aarch64.rs index 98a9366a3..ada056c09 100644 --- a/litebox_syscall_rewriter/src/aarch64.rs +++ b/litebox_syscall_rewriter/src/aarch64.rs @@ -970,60 +970,44 @@ const GUEST_X18_OFFSET_PLACEHOLDER: u16 = MAX_GUEST_TPIDR_OFFSET - 8; pub const X18_FRAME_BYTES: u16 = 16; const X18_FRAME_OFF_SCRATCHES: u16 = 0; -/// State of an x18 gate's scratch frame at an interrupted instruction boundary. +/// Location of guest scratch state at an emitted x18 instruction boundary. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum X18FrameState { - /// No frame exists; SP and scratch registers are already guest-visible. Absent, - /// SP points to the frame, but the original scratch registers are still live. AtSpRegistersLive, - /// SP points to the frame holding the original scratch registers. AtSpRestoreRegisters, - /// SP is above the frame, which still holds the original scratch registers. BelowSpRestoreRegisters, } /// Source of the guest-visible x18 value during recovery. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum X18ValueSource { - /// Read x18 from its runtime-owned logical slot. Slot, - /// Read the updated x18 value from the gate's value scratch register. Scratch, } /// Guest PC selected after x18 gate recovery. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum X18Resume { - /// Resume at the original guest instruction. Original, - /// Resume at the instruction following the original guest instruction. Next, - /// Resume at the original conditional branch target. TakenTarget, } -/// Runtime-owned x18-slot access about to execute at a gate boundary. +/// Runtime-owned x18-slot access at an emitted instruction boundary. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum X18SlotAccess { - /// The boundary instruction does not access the logical x18 slot. None, - /// The boundary instruction loads the logical x18 slot. Load, - /// The boundary instruction stores the logical x18 slot. Store, } /// Host-neutral recovery semantics derived from an emitted x18 gate boundary. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct X18RecoveryPlan { - /// Scratch-frame state at this instruction boundary. pub frame: X18FrameState, - /// Source of the canonical guest x18 value. pub value: X18ValueSource, - /// Guest PC selected by recovery. pub resume: X18Resume, - /// Runtime-owned slot access about to execute at this boundary. pub slot_access: X18SlotAccess, } @@ -1160,9 +1144,15 @@ pub enum X18CompareBranchOffset { TakenPop = 36, /// Branch to the original taken target. TakenBranch = 40, + /// First byte after the executable gate body. + ExecutableEnd = 44, } impl X18CompareBranchOffset { + const fn as_usize(self) -> usize { + self as u8 as usize + } + /// Decodes an executable instruction boundary. pub fn from_offset(offset: usize) -> Option { Some(match offset { @@ -1182,13 +1172,13 @@ impl X18CompareBranchOffset { } /// Returns the recovery semantics for this emitted instruction boundary. - pub const fn recovery_plan(self) -> X18RecoveryPlan { + pub const fn recovery_plan(self) -> Option { use X18FrameState::{Absent, AtSpRegistersLive, AtSpRestoreRegisters}; use X18Resume::{Next, Original, TakenTarget}; use X18SlotAccess::{Load, None as NoSlotAccess}; use X18ValueSource::Slot; - match self { + Some(match self { Self::Entry => X18RecoveryPlan::new(Absent, Slot, Original, NoSlotAccess), Self::Spill => X18RecoveryPlan::new(AtSpRegistersLive, Slot, Original, NoSlotAccess), Self::Anchor | Self::Test => { @@ -1209,7 +1199,8 @@ impl X18CompareBranchOffset { X18RecoveryPlan::new(AtSpRegistersLive, Slot, TakenTarget, NoSlotAccess) } Self::TakenBranch => X18RecoveryPlan::new(Absent, Slot, TakenTarget, NoSlotAccess), - } + Self::ExecutableEnd => return None, + }) } } @@ -1231,9 +1222,15 @@ pub enum X18AdrOffset { Restore = 20, /// Branch back to guest code. Return = 24, + /// First byte after the executable gate body. + ExecutableEnd = 28, } impl X18AdrOffset { + const fn as_usize(self) -> usize { + self as u8 as usize + } + /// Decodes an executable instruction boundary. pub fn from_offset(offset: usize) -> Option { Some(match offset { @@ -1249,13 +1246,13 @@ impl X18AdrOffset { } /// Returns the recovery semantics for this emitted instruction boundary. - pub const fn recovery_plan(self) -> X18RecoveryPlan { + pub const fn recovery_plan(self) -> Option { use X18FrameState::{Absent, AtSpRegistersLive, AtSpRestoreRegisters}; use X18Resume::{Next, Original}; use X18SlotAccess::{None as NoSlotAccess, Store}; use X18ValueSource::{Scratch, Slot}; - match self { + Some(match self { Self::Entry => X18RecoveryPlan::new(Absent, Slot, Original, NoSlotAccess), Self::Anchor => X18RecoveryPlan::new(AtSpRegistersLive, Slot, Original, NoSlotAccess), Self::Adrp | Self::Add => { @@ -1264,7 +1261,8 @@ impl X18AdrOffset { Self::SlotStore => X18RecoveryPlan::new(AtSpRestoreRegisters, Scratch, Next, Store), Self::Restore => X18RecoveryPlan::new(AtSpRestoreRegisters, Slot, Next, NoSlotAccess), Self::Return => X18RecoveryPlan::new(Absent, Slot, Next, NoSlotAccess), - } + Self::ExecutableEnd => return None, + }) } } @@ -1288,6 +1286,162 @@ pub const SVC_FRAME_OFF_RETADDR: u16 = 8; /// branches here to resume at the original syscall site. pub const SVC_FRAME_OFF_STUB: u16 = 16; +/// Whether a synchronous fault at a boundary is attributable to runtime state. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RuntimeAccess { + NoAccess, + Memory, +} + +/// Source of the guest-visible MRS destination value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MrsTpidrValueSource { + Register, + Slot, +} + +/// Host-neutral recovery semantics for an interrupted MRS-TPIDR gate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MrsTpidrRecoveryPlan { + pub value: MrsTpidrValueSource, + pub completed: bool, + pub runtime_access: RuntimeAccess, +} + +/// Instruction boundaries in the emitted MRS-TPIDR gate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum MrsTpidrGateOffset { + Entry = 0, + SlotLoad = 4, + Return = 8, + ExecutableEnd = 12, +} + +impl MrsTpidrGateOffset { + const fn as_usize(self) -> usize { + self as u8 as usize + } + + pub fn from_offset(offset: usize) -> Option { + Some(match offset { + 0 => Self::Entry, + 4 => Self::SlotLoad, + 8 => Self::Return, + _ => return None, + }) + } + + pub const fn recovery_plan(self) -> Option { + Some(match self { + Self::Entry => MrsTpidrRecoveryPlan { + value: MrsTpidrValueSource::Register, + completed: false, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::SlotLoad => MrsTpidrRecoveryPlan { + value: MrsTpidrValueSource::Slot, + completed: true, + runtime_access: RuntimeAccess::Memory, + }, + Self::Return => MrsTpidrRecoveryPlan { + value: MrsTpidrValueSource::Register, + completed: true, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::ExecutableEnd => return None, + }) + } +} + +/// Location of guest x16 and SP at an emitted SVC instruction boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SvcFrameState { + NoFrame, + FrameAtSpX16Live, + RestoreX16FromFrame, +} + +/// Host-neutral recovery semantics for an interrupted SVC gate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SvcRecoveryPlan { + pub frame: SvcFrameState, + pub runtime_access: RuntimeAccess, +} + +/// Instruction boundaries in the emitted SVC gate and its outbound stub. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum SvcGateOffset { + Entry = 0, + SpillX16 = 4, + MaterializeReturnPage = 8, + MaterializeReturnAddress = 12, + StoreReturnAddress = 16, + MaterializeOutboundStub = 20, + StoreOutboundStub = 24, + LoadCallback = 28, + BranchCallback = 32, + OutboundRestoreX16 = 36, + OutboundRestoreSp = 40, + Return = 44, + ExecutableEnd = 48, +} + +impl SvcGateOffset { + const fn as_usize(self) -> usize { + self as u8 as usize + } + + pub fn from_offset(offset: usize) -> Option { + Some(match offset { + 0 => Self::Entry, + 4 => Self::SpillX16, + 8 => Self::MaterializeReturnPage, + 12 => Self::MaterializeReturnAddress, + 16 => Self::StoreReturnAddress, + 20 => Self::MaterializeOutboundStub, + 24 => Self::StoreOutboundStub, + 28 => Self::LoadCallback, + 32 => Self::BranchCallback, + 36 => Self::OutboundRestoreX16, + 40 => Self::OutboundRestoreSp, + 44 => Self::Return, + _ => return None, + }) + } + + pub const fn recovery_plan(self) -> Option { + use RuntimeAccess::{Memory, NoAccess}; + Some(match self { + Self::Entry => SvcRecoveryPlan { + frame: SvcFrameState::NoFrame, + runtime_access: NoAccess, + }, + Self::SpillX16 | Self::MaterializeReturnPage => SvcRecoveryPlan { + frame: SvcFrameState::FrameAtSpX16Live, + runtime_access: NoAccess, + }, + Self::MaterializeReturnAddress + | Self::StoreReturnAddress + | Self::MaterializeOutboundStub + | Self::StoreOutboundStub + | Self::BranchCallback => SvcRecoveryPlan { + frame: SvcFrameState::RestoreX16FromFrame, + runtime_access: NoAccess, + }, + Self::LoadCallback => SvcRecoveryPlan { + frame: SvcFrameState::RestoreX16FromFrame, + runtime_access: Memory, + }, + Self::OutboundRestoreX16 + | Self::OutboundRestoreSp + | Self::Return + | Self::ExecutableEnd => return None, + }) + } +} + pub const RT_SIGRETURN_TRAMPOLINE_BYTES: usize = 48; /// Emits a synthetic AArch64 `rt_sigreturn` restorer that dispatches through @@ -1377,7 +1531,115 @@ pub const MSR_FRAME_BYTES: u16 = 32; const MSR_FRAME_OFF_X16: u16 = 0; /// Captured guest thread-pointer value, staged while all guest registers are /// still pristine. -const MSR_FRAME_OFF_VALUE: u16 = 16; +pub const MSR_FRAME_OFF_VALUE: u16 = 16; + +/// Location of guest scratch state at an emitted MSR-TPIDR instruction boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MsrTpidrFrameState { + Absent, + RegistersLive, + RestoreRegisters, +} + +/// Host-neutral recovery semantics for an interrupted MSR-TPIDR gate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MsrTpidrRecoveryPlan { + pub frame: MsrTpidrFrameState, + /// Whether the staged source value must authenticate recovery. + pub validate_capture: bool, + pub completed: bool, + pub runtime_access: RuntimeAccess, +} + +/// Instruction boundaries in the emitted MSR-TPIDR gate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum MsrTpidrGateOffset { + Entry = 0, + SpillRegisters = 4, + CaptureSource = 8, + ReadAnchor = 12, + LoadCapturedSource = 16, + SlotStore = 20, + RestoreRegisters = 24, + RestoreSp = 28, + Return = 32, + ExecutableEnd = 36, +} + +impl MsrTpidrGateOffset { + const fn as_usize(self) -> usize { + self as u8 as usize + } + + pub fn from_offset(offset: usize) -> Option { + Some(match offset { + 0 => Self::Entry, + 4 => Self::SpillRegisters, + 8 => Self::CaptureSource, + 12 => Self::ReadAnchor, + 16 => Self::LoadCapturedSource, + 20 => Self::SlotStore, + 24 => Self::RestoreRegisters, + 28 => Self::RestoreSp, + 32 => Self::Return, + _ => return None, + }) + } + + pub const fn recovery_plan(self) -> Option { + use MsrTpidrFrameState::{Absent, RegistersLive, RestoreRegisters}; + Some(match self { + Self::Entry => MsrTpidrRecoveryPlan { + frame: Absent, + validate_capture: false, + completed: false, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::SpillRegisters | Self::CaptureSource => MsrTpidrRecoveryPlan { + frame: RegistersLive, + validate_capture: false, + completed: false, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::ReadAnchor => MsrTpidrRecoveryPlan { + frame: RegistersLive, + validate_capture: true, + completed: false, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::LoadCapturedSource | Self::SlotStore => MsrTpidrRecoveryPlan { + frame: RestoreRegisters, + validate_capture: true, + completed: false, + runtime_access: if matches!(self, Self::SlotStore) { + RuntimeAccess::Memory + } else { + RuntimeAccess::NoAccess + }, + }, + Self::RestoreRegisters => MsrTpidrRecoveryPlan { + frame: RestoreRegisters, + validate_capture: true, + completed: true, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::RestoreSp => MsrTpidrRecoveryPlan { + frame: RegistersLive, + validate_capture: false, + completed: true, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::Return => MsrTpidrRecoveryPlan { + frame: Absent, + validate_capture: false, + completed: true, + runtime_access: RuntimeAccess::NoAccess, + }, + Self::ExecutableEnd => return None, + }) + } +} // --- Trampoline layout offsets (all in bytes) --- @@ -3165,16 +3427,14 @@ impl GateMetadata { /// classified as one. pub(crate) const fn executable_end(self) -> usize { match self { - // The `B` back to the original site at 44 is the last instruction. - GateMetadata::Svc => 48, - // `MRS`, the guest-TLS `LDR`, then the `B` back. - GateMetadata::MrsTpidr { .. } => 12, - // Frame teardown ends at 28, then the `B` back at 32. The second - // `B` at 36 never executes, so a PC there is not a live gate PC. - GateMetadata::MsrTpidr { .. } => 36, + GateMetadata::Svc => SvcGateOffset::ExecutableEnd.as_usize(), + GateMetadata::MrsTpidr { .. } => MrsTpidrGateOffset::ExecutableEnd.as_usize(), + GateMetadata::MsrTpidr { .. } => MsrTpidrGateOffset::ExecutableEnd.as_usize(), GateMetadata::X18 { .. } => X18GateOffset::ExecutableEnd.as_usize(), - GateMetadata::X18CompareBranch { .. } => 44, - GateMetadata::X18Adr { .. } => 28, + GateMetadata::X18CompareBranch { .. } => { + X18CompareBranchOffset::ExecutableEnd.as_usize() + } + GateMetadata::X18Adr { .. } => X18AdrOffset::ExecutableEnd.as_usize(), } } @@ -3182,34 +3442,14 @@ impl GateMetadata { /// original site. pub(crate) const fn return_offset(self) -> usize { match self { - GateMetadata::Svc => 44, - GateMetadata::MrsTpidr { .. } => 8, - GateMetadata::MsrTpidr { .. } => 32, + GateMetadata::Svc => SvcGateOffset::Return.as_usize(), + GateMetadata::MrsTpidr { .. } => MrsTpidrGateOffset::Return.as_usize(), + GateMetadata::MsrTpidr { .. } => MsrTpidrGateOffset::Return.as_usize(), GateMetadata::X18 { .. } => X18GateOffset::Return.as_usize(), - GateMetadata::X18CompareBranch { .. } => 28, - GateMetadata::X18Adr { .. } => 24, - } - } - - /// Byte offset of the instruction that commits the gate's architectural - /// effect. A saved PC names the instruction about to execute, so past this - /// offset the effect has happened. - /// - /// This says when the effect lands, not whether signal canonicalization - /// rewinds or carries the gate forward. That decision also depends on the - /// gate kind and recoverable spill-frame state. - pub const fn commit_offset(self) -> usize { - match self { - // The shim performs the syscall, so any PC in an `SVC` slot is - // still pre-commit — though at 4 or beyond `SP` and `X16` still - // need undoing. - GateMetadata::Svc => 0, - // The guest-TLS load. - GateMetadata::MrsTpidr { .. } => 4, - // The guest-TLS store. - GateMetadata::MsrTpidr { .. } => 20, - GateMetadata::X18 { .. } => X18GateOffset::Transform.as_usize(), - GateMetadata::X18CompareBranch { .. } | GateMetadata::X18Adr { .. } => 16, + GateMetadata::X18CompareBranch { .. } => { + X18CompareBranchOffset::FallthroughBranch.as_usize() + } + GateMetadata::X18Adr { .. } => X18AdrOffset::Return.as_usize(), } } } @@ -3220,14 +3460,12 @@ pub fn decode_gate_metadata_word(word: u32) -> Option { } /// A validated gate slot containing some PC, with what a signal handler needs -/// to canonicalize the interrupted context: where the slot starts, how far -/// into it the guest-visible effect commits, and which guest instruction it -/// replaced. +/// to canonicalize the interrupted context: where the slot starts and which +/// guest instruction it replaced. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ClassifiedGate { slot_offset: usize, slot_size: u8, - commit_offset: u8, anchor_scratch: u8, original_site: u64, conditional_target: u64, @@ -3245,11 +3483,6 @@ impl ClassifiedGate { self.slot_size } - /// Template-derived architectural commit offset. - pub fn commit_offset(self) -> u8 { - self.commit_offset - } - pub fn anchor_scratch(self) -> Option { matches!( self.metadata, @@ -3312,8 +3545,15 @@ pub fn classify_copied_gate_slot(slot: &[u8], slot_vaddr: u64, pc: u64) -> Optio } let trampoline_base = match metadata { GateMetadata::Svc => decode_ldr_literal_target( - u32::from_le_bytes(slot.get(28..32)?.try_into().ok()?), - slot_vaddr + 28, + u32::from_le_bytes( + slot.get( + SvcGateOffset::LoadCallback.as_usize() + ..SvcGateOffset::LoadCallback.as_usize() + INSN_BYTES, + )? + .try_into() + .ok()?, + ), + slot_vaddr + SvcGateOffset::LoadCallback.as_usize() as u64, )?, GateMetadata::MrsTpidr { .. } | GateMetadata::MsrTpidr { .. } @@ -3347,8 +3587,13 @@ pub fn classify_copied_gate_slot(slot: &[u8], slot_vaddr: u64, pc: u64) -> Optio )?; let conditional_target = if matches!(metadata, GateMetadata::X18CompareBranch { .. }) { decode_branch_target( - u32::from_le_bytes(slot[40..44].try_into().ok()?), - slot_vaddr + 40, + u32::from_le_bytes( + slot[X18CompareBranchOffset::TakenBranch.as_usize() + ..X18CompareBranchOffset::TakenBranch.as_usize() + INSN_BYTES] + .try_into() + .ok()?, + ), + slot_vaddr + X18CompareBranchOffset::TakenBranch.as_usize() as u64, )? } else { 0 @@ -3356,7 +3601,6 @@ pub fn classify_copied_gate_slot(slot: &[u8], slot_vaddr: u64, pc: u64) -> Optio Some(ClassifiedGate { slot_offset: 0, slot_size: u8::try_from(slot.len()).ok()?, - commit_offset: u8::try_from(metadata.commit_offset()).ok()?, anchor_scratch: match metadata { GateMetadata::X18 { .. } | GateMetadata::X18CompareBranch { .. } @@ -3423,8 +3667,13 @@ fn classify_gate_pc_with_candidates( )?; let conditional_target = if matches!(metadata, GateMetadata::X18CompareBranch { .. }) { decode_branch_target( - u32::from_le_bytes(slot[40..44].try_into().ok()?), - slot_vaddr + 40, + u32::from_le_bytes( + slot[X18CompareBranchOffset::TakenBranch.as_usize() + ..X18CompareBranchOffset::TakenBranch.as_usize() + INSN_BYTES] + .try_into() + .ok()?, + ), + slot_vaddr + X18CompareBranchOffset::TakenBranch.as_usize() as u64, )? } else { 0 @@ -3432,7 +3681,6 @@ fn classify_gate_pc_with_candidates( match_found = Some(ClassifiedGate { slot_offset: start, slot_size: u8::try_from(slot_size).ok()?, - commit_offset: u8::try_from(metadata.commit_offset()).ok()?, anchor_scratch: match metadata { GateMetadata::X18 { .. } | GateMetadata::X18CompareBranch { .. } @@ -5489,7 +5737,6 @@ mod tests { Some(ClassifiedGate { slot_offset: start, slot_size: u8::try_from(size).unwrap(), - commit_offset: u8::try_from(metadata.commit_offset()).unwrap(), anchor_scratch: 0, original_site: 0x1000 + match metadata {