diff --git a/hfo2/src/abi.rs b/hfo2/src/abi.rs index c27569f83..8f803a2c7 100644 --- a/hfo2/src/abi.rs +++ b/hfo2/src/abi.rs @@ -115,3 +115,84 @@ impl TryFrom for HfShare { } } } + +#[cfg(test)] +mod test { + use super::*; + + /// Encode a preempted response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_preempted() { + let res = HfVCpuRunReturn::Preempted; + assert_eq!(res.into_raw(), 0); + } + + /// Encode a yield response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_yield() { + let res = HfVCpuRunReturn::Yield; + assert_eq!(res.into_raw(), 1); + } + + /// Encode wait-for-interrupt response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_interrupt() { + let res = HfVCpuRunReturn::WaitForInterrupt { + ns: HF_SLEEP_INDEFINITE, + }; + assert_eq!(res.into_raw(), 0xffffffffffffff02); + } + + /// Encoding wait-for-interrupt response with too large sleep duration will + /// drop the top octet. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_interrupt_sleep_too_long() { + let res = HfVCpuRunReturn::WaitForInterrupt { + ns: 0xcc22888888888888, + }; + assert_eq!(res.into_raw(), 0x2288888888888802); + } + + /// Encode wait-for-message response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_message() { + let res = HfVCpuRunReturn::WaitForMessage { + ns: HF_SLEEP_INDEFINITE, + }; + assert_eq!(res.into_raw(), 0xffffffffffffff03); + } + + /// Encoding wait-for-message response with too large sleep duration will + /// drop the top octet. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_message_sleep_too_long() { + let res = HfVCpuRunReturn::WaitForMessage { + ns: 0xaa99777777777777, + }; + assert_eq!(res.into_raw(), 0x9977777777777703); + } + + /// Encode wake up response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_wake_up() { + let res = HfVCpuRunReturn::WakeUp { + vm_id: 0x1234, + vcpu: 0xabcd, + }; + assert_eq!(res.into_raw(), 0x1234abcd0004); + } + + /// Encode a 'notify waiters' response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_notify_waiters() { + let res = HfVCpuRunReturn::NotifyWaiters; + assert_eq!(res.into_raw(), 6); + } + + /// Encode an aborted response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_aborted() { + let res = HfVCpuRunReturn::Aborted; + assert_eq!(res.into_raw(), 7); + } +} diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index da25bc2d6..0dbdaf3aa 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -24,6 +24,7 @@ use crate::abi::*; use crate::addr::*; use crate::arch::*; use crate::cpu::*; +use crate::init::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; @@ -47,19 +48,6 @@ use crate::vm::*; // of a page. const_assert_eq!(hf_mailbox_size; HF_MAILBOX_SIZE, PAGE_SIZE); -/// A global page pool for sharing memories. Its mutability is needed only for -/// initialization. -static mut API_PAGE_POOL: MaybeUninit = MaybeUninit::uninit(); - -/// Initialises the API page pool by taking ownership of the contents of the -/// given page pool. -/// TODO(HfO2): The ownership of `ppool` is actually moved from `one_time_init` -/// to here. Refactor this function like `Api::new(ppool: MPool) -> Api`. (#31) -#[no_mangle] -pub unsafe extern "C" fn api_init(ppool: *const MPool) { - API_PAGE_POOL = MaybeUninit::new(MPool::new_from(&*ppool)); -} - /// Switches the physical CPU back to the corresponding vcpu of the primary VM. /// /// This triggers the scheduling logic to run. Run in the context of secondary @@ -70,11 +58,11 @@ unsafe fn switch_to_primary( mut primary_ret: HfVCpuRunReturn, secondary_state: VCpuStatus, ) -> *mut VCpu { - let primary = vm_find(HF_PRIMARY_VM_ID); - let next = vm_get_vcpu( - primary, - cpu_index(current.get_inner().cpu) as spci_vcpu_index_t, - ); + let primary = hafnium().vm_manager.get(HF_PRIMARY_VM_ID).unwrap(); + let next = primary + .vcpus + .get(cpu_index(&*current.get_inner().cpu)) + .unwrap(); // If the secondary is blocked but has a timer running, sleep until the // timer fires rather than indefinitely. @@ -94,8 +82,7 @@ unsafe fn switch_to_primary( // The use of `get_mut_unchecked()` is safe because the currently running pCPU implicitly owns // `next`. Notice that `next` is the vCPU of the primary VM that corresponds to the currently // running pCPU. - (*next) - .inner + next.inner .get_mut_unchecked() .regs .set_retval(primary_ret.into_raw()); @@ -103,7 +90,7 @@ unsafe fn switch_to_primary( // Mark the current vcpu as waiting. current.get_inner_mut().state = secondary_state; - next + next as *const _ as usize as *mut _ } /// Returns to the primary vm and signals that the vcpu still has work to do so. @@ -225,19 +212,14 @@ pub unsafe extern "C" fn api_vcpu_get_count( vm_id: spci_vm_id_t, current: *const VCpu, ) -> spci_vcpu_count_t { - let vm; - // Only the primary VM needs to know about vcpus for scheduling. if (*(*current).vm).id != HF_PRIMARY_VM_ID { return 0; } - vm = vm_find(vm_id); - if vm.is_null() { - return 0; - } + let vm = unwrap_or!(hafnium().vm_manager.get(vm_id), return 0); - (*vm).vcpu_count + vm.vcpus.len() as spci_vcpu_count_t } /// This function is called by the architecture-specific context switching @@ -286,7 +268,7 @@ unsafe fn internal_interrupt_inject( /// `vcpu.execution_lock` has acquired. unsafe fn vcpu_prepare_run( current: &VCpuExecutionLocked, - vcpu: *mut VCpu, + vcpu: &VCpu, run_ret: HfVCpuRunReturn, ) -> Result { let mut vcpu_inner = (*vcpu).inner.try_lock().map_err(|_| { @@ -300,13 +282,9 @@ unsafe fn vcpu_prepare_run( run_ret })?; - if (*(*vcpu).vm).aborting.load(Ordering::Relaxed) { + if (*vcpu.vm).aborting.load(Ordering::Relaxed) { if vcpu_inner.state != VCpuStatus::Aborted { - dlog!( - "Aborting VM {} vCPU {}\n", - (*(*vcpu).vm).id, - vcpu_index(vcpu) - ); + dlog!("Aborting VM {} vCPU {}\n", (*vcpu.vm).id, vcpu_index(vcpu)); vcpu_inner.state = VCpuStatus::Aborted; } return Err(run_ret); @@ -324,13 +302,13 @@ unsafe fn vcpu_prepare_run( // when it is going to be needed. This ensures there are no inter-vCPU // dependencies in the common run case meaning the sensitive context // switch performance is consistent. - VCpuStatus::BlockedMailbox if (*(*vcpu).vm).inner.lock().try_read().is_ok() => { + VCpuStatus::BlockedMailbox if (*vcpu.vm).inner.lock().try_read().is_ok() => { vcpu_inner.regs.set_retval(SpciReturn::Success as uintreg_t); } // Allow virtual interrupts to be delivered. VCpuStatus::BlockedMailbox | VCpuStatus::BlockedInterrupt - if (*vcpu).interrupts.lock().is_interrupted() => + if vcpu.interrupts.lock().is_interrupted() => { // break; } @@ -395,18 +373,12 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested VM must exist. - let vm = vm_find(vm_id); - if vm.is_null() { - return ret.into_raw(); - } + let vm = unwrap_or!(hafnium().vm_manager.get(vm_id), return ret.into_raw()); // The requested vcpu must exist. - if vcpu_idx >= (*vm).vcpu_count { - return ret.into_raw(); - } + let vcpu = unwrap_or!(vm.vcpus.get(vcpu_idx as usize), return ret.into_raw()); // Update state if allowed. - let vcpu = vm_get_vcpu(vm, vcpu_idx); let mut vcpu_locked = match vcpu_prepare_run(¤t, vcpu, ret) { Ok(locked) => locked, Err(ret) => return ret.into_raw(), @@ -493,10 +465,7 @@ pub unsafe extern "C" fn api_vm_configure( // TODO: the scope of the can be reduced but will require restructing // to keep a single unlock point. let mut vm_inner = (*vm).inner.lock(); - if vm_inner - .configure(send, recv, API_PAGE_POOL.get_ref()) - .is_err() - { + if vm_inner.configure(send, recv, &hafnium().mpool).is_err() { return -1; } @@ -550,10 +519,10 @@ pub unsafe extern "C" fn api_spci_msg_send( } // Ensure the target VM exists. - let to = vm_find(from_msg_replica.target_vm_id); - if to.is_null() { - return SpciReturn::InvalidParameters; - } + let to = unwrap_or!( + hafnium().vm_manager.get(from_msg_replica.target_vm_id), + return SpciReturn::InvalidParameters + ); // Hf needs to hold the lock on `to` before the mailbox state is checked. // The lock on `to` must be held until the information is copied to `to` Rx @@ -616,7 +585,7 @@ pub unsafe extern "C" fn api_spci_msg_send( // fields in message_buffer. The memory area message_buffer must be // exclusively owned by Hf so that TOCTOU issues do not arise. let ret = spci_msg_handle_architected_message( - &ManuallyDrop::new(VmLocked::from_raw(to)), + &ManuallyDrop::new(VmLocked::from_raw(to as *const _ as usize as *mut _)), &ManuallyDrop::new(VmLocked::from_raw(from)), architected_message_replica, &from_msg_replica, @@ -730,10 +699,7 @@ pub unsafe extern "C" fn api_mailbox_waiter_get(vm_id: spci_vm_id_t, current: *c return -1; } - let vm = vm_find(vm_id); - if vm.is_null() { - return -1; - } + let vm = unwrap_or!(hafnium().vm_manager.get(vm_id), return -1); // Check if there are outstanding notifications from given vm. let entry = (*vm).inner.lock().fetch_waiter(); @@ -833,26 +799,17 @@ pub unsafe extern "C" fn api_interrupt_inject( next: *mut *mut VCpu, ) -> i64 { let mut current = ManuallyDrop::new(VCpuExecutionLocked::from_raw(current)); - let target_vm = vm_find(target_vm_id); + let target_vm = unwrap_or!(hafnium().vm_manager.get(target_vm_id), return -1); if intid >= HF_NUM_INTIDS { return -1; } - if target_vm.is_null() { - return -1; - } - - if target_vcpu_idx >= (*target_vm).vcpu_count { - // The requested vcpu must exist. - return -1; - } - if !is_injection_allowed(target_vm_id, current.deref()) { return -1; } - let target_vcpu = vm_get_vcpu(target_vm, target_vcpu_idx); + let target_vcpu = unwrap_or!(target_vm.vcpus.get(target_vcpu_idx as usize), return -1); dlog!( "Injecting IRQ {} for VM {} VCPU {} from VM {} VCPU {}\n", @@ -868,7 +825,7 @@ pub unsafe extern "C" fn api_interrupt_inject( /// Clears a region of physical memory by overwriting it with zeros. The data is /// flushed from the cache so the memory has been cleared across the system. fn clear_memory(begin: paddr_t, end: paddr_t, ppool: &MPool) -> Result<(), ()> { - let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); + let mut hypervisor_ptable = hafnium().memory_manager.hypervisor_ptable.lock(); let size = pa_difference(begin, end); let region = pa_addr(begin); @@ -925,7 +882,7 @@ pub fn spci_share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool: MPool = MPool::new_with_fallback(unsafe { API_PAGE_POOL.get_ref() }); + let local_page_pool: MPool = MPool::new_with_fallback(&hafnium().mpool); // Obtain the single contiguous set of pages from the memory_region. // TODO: Add support for multiple constituent regions. @@ -937,7 +894,7 @@ pub fn spci_share_memory( // Check if the state transition is lawful for both VMs involved in the // memory exchange, ensure that all constituents of a memory region being // shared are at the same state. - let (orig_from_mode, from_mode, to_mode) = ok_or_return!( + let (orig_from_mode, from_mode, to_mode) = ok_or!( spci_msg_check_transition( &to_locked, &from_locked, @@ -946,7 +903,7 @@ pub fn spci_share_memory( end, memory_to_attributes, ), - SpciReturn::InvalidParameters + return SpciReturn::InvalidParameters ); let pa_begin = pa_from_ipa(begin); @@ -1006,12 +963,7 @@ fn share_memory( } // Ensure the target VM exists. - let to = unsafe { vm_find(vm_id) }; - if to.is_null() { - return Err(()); - } - - let to = unsafe { &*to }; + let to = hafnium().vm_manager.get(vm_id).ok_or(())?; let begin = addr; let end = ipa_add(addr, size); @@ -1036,7 +988,7 @@ fn share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool = MPool::new_with_fallback(unsafe { API_PAGE_POOL.get_ref() }); + let local_page_pool = MPool::new_with_fallback(&hafnium().mpool); let (mut from_inner, mut to_inner) = SpinLock::lock_both(&(*from).inner, &(*to).inner); @@ -1115,7 +1067,7 @@ pub unsafe extern "C" fn api_share_memory( ) -> i64 { // Convert the sharing request to memory management modes. // The input is untrusted so might not be a valid value. - let share = ok_or_return!(HfShare::try_from(share), -1); + let share = ok_or!(HfShare::try_from(share), return -1); match share_memory(vm_id, addr, size, share, &*current) { Ok(_) => 0, diff --git a/hfo2/src/arch/fake.rs b/hfo2/src/arch/fake.rs index d8d5d9b52..4079f748a 100644 --- a/hfo2/src/arch/fake.rs +++ b/hfo2/src/arch/fake.rs @@ -17,6 +17,9 @@ // TODO: Refactor type names and remove this. #![allow(non_camel_case_types)] +use crate::boot_params::*; +use crate::mm::*; +use crate::mpool::*; use crate::types::*; /// The integer type corresponding to the native register size. @@ -51,3 +54,29 @@ pub struct ArchRegs { pub fn arch_cpu_module_init() { // Do nothing. } + +// TODO(HfO2): Following functions are empty, since linker complains if the +// implementations of functions are missing even they're never called in the +// unit tests. Make a custom target and remove those (#46.) +#[no_mangle] +pub extern "C" fn arch_one_time_init() { + unreachable!(); +} + +#[no_mangle] +pub fn plat_get_boot_params( + _stage1_locked: mm_stage1_locked, + _p: *mut BootParams, + _ppool: *mut MPool, +) -> bool { + unreachable!(); +} + +#[no_mangle] +pub fn plat_update_boot_params( + _stage1_locked: mm_stage1_locked, + _p: *mut BootParamsUpdate, + _ppool: *mut MPool, +) -> bool { + unreachable!(); +} diff --git a/hfo2/src/arch/mod.rs b/hfo2/src/arch/mod.rs index 08b4b7ac6..ae2d204be 100644 --- a/hfo2/src/arch/mod.rs +++ b/hfo2/src/arch/mod.rs @@ -14,7 +14,10 @@ * limitations under the License. */ +#[cfg(target_arch = "aarch64")] mod aarch64; + +#[cfg(target_arch = "x86_64")] mod fake; #[cfg(target_arch = "aarch64")] diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs new file mode 100644 index 000000000..b969a3877 --- /dev/null +++ b/hfo2/src/boot_params.rs @@ -0,0 +1,115 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem::MaybeUninit; + +use crate::addr::*; +use crate::arch::*; +use crate::mm::*; +use crate::mpool::*; +use crate::spinlock::*; +use crate::types::*; + +pub const MAX_MEM_RANGES: usize = 20; + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct MemRange { + pub begin: paddr_t, + pub end: paddr_t, +} + +impl MemRange { + pub fn new(begin: paddr_t, end: paddr_t) -> Self { + Self { begin, end } + } +} + +#[repr(C)] +pub struct BootParams { + pub cpu_ids: [cpu_id_t; MAX_CPUS], + pub cpu_count: usize, + pub mem_ranges: [MemRange; MAX_MEM_RANGES], + pub mem_ranges_count: usize, + pub initrd_begin: paddr_t, + pub initrd_end: paddr_t, + pub kernel_arg: uintreg_t, +} + +#[repr(C)] +pub struct BootParamsUpdate { + pub reserved_ranges: [MemRange; MAX_MEM_RANGES], + pub reserved_ranges_count: usize, + pub initrd_begin: paddr_t, + pub initrd_end: paddr_t, +} + +impl BootParamsUpdate { + pub fn new(initrd_begin: paddr_t, initrd_end: paddr_t) -> Self { + Self { + reserved_ranges: [MemRange::new(pa_init(0), pa_init(0)); MAX_MEM_RANGES], + reserved_ranges_count: 0, + initrd_begin, + initrd_end, + } + } +} + +/// TODO(HfO2): `plat.c`, containing those functions are not ported into Rust. +/// It's because functions in `plat.c` are denoted by `#pragma weak` which is +/// not supported in Rust yet. (#47.) +extern "C" { + fn plat_get_boot_params( + stage1_locked: mm_stage1_locked, + p: *mut BootParams, + ppool: *mut MPool, + ) -> bool; + + fn plat_update_boot_params( + stage1_locked: mm_stage1_locked, + p: *mut BootParamsUpdate, + ppool: *mut MPool, + ) -> bool; +} + +/// Reads platform-specific boot parameters. +pub fn boot_params_get( + ptable: &mut SpinLockGuard>, + ppool: &mut MPool, +) -> Option { + unsafe { + let mut p: MaybeUninit = MaybeUninit::uninit(); + + if plat_get_boot_params(mm_stage1_locked::from_ref(ptable), p.get_mut(), ppool) { + Some(p.assume_init()) + } else { + None + } + } +} + +/// Updates boot parameters for primary VM to read. +pub fn boot_params_update( + ptable: &mut SpinLockGuard>, + p: &mut BootParamsUpdate, + ppool: &mut MPool, +) -> Result<(), ()> { + if unsafe { plat_update_boot_params(mm_stage1_locked::from_ref(ptable), p, ppool) } { + Ok(()) + } else { + Err(()) + } +} diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 38b450af9..954c35a27 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -20,13 +20,15 @@ use core::ptr; use crate::addr::*; use crate::arch::*; +use crate::init::*; use crate::mm::*; use crate::page::*; use crate::spinlock::*; -use crate::std::*; use crate::types::*; use crate::vm::*; +use arrayvec::ArrayVec; + /// From inc/hf/arch/cpu.h. extern "C" { /// Disables interrupts. @@ -59,7 +61,7 @@ extern "C" { fn arch_regs_set_retval(r: *mut ArchRegs, v: uintreg_t); } -const STACK_SIZE: usize = PAGE_SIZE; +pub const STACK_SIZE: usize = PAGE_SIZE; /// The number of bits in each element of the interrupt bitfields. pub const INTERRUPT_REGISTER_BITS: usize = 32; @@ -97,6 +99,14 @@ pub struct Interrupts { } impl Interrupts { + pub fn new() -> Self { + Self { + enabled: [0; HF_NUM_INTIDS as usize / INTERRUPT_REGISTER_BITS], + pending: [0; HF_NUM_INTIDS as usize / INTERRUPT_REGISTER_BITS], + enabled_and_pending_count: 0, + } + } + pub fn id_to_index(intid: intid_t) -> Result<(usize, u32), ()> { if intid >= HF_NUM_INTIDS { return Err(()); @@ -191,6 +201,25 @@ impl Interrupts { } impl ArchRegs { + pub fn new() -> Self { + // TODO(HfO2): Originally, ArchRegs are filled by 0 when they're crated. + // However, doing like + // ``` + // let mut ret = MaybeUninit::uninit().assume_init(); + // memset(&mut ret, 0, mem::size_of_val(&ret)); + // ``` + // allocates large memory in stack to hold the value of ArchRegs, and + // introduce unnecessary memcpy. + // Note that MaybeUninit::zeroed() makes same assembly. I guess Rust + // native MaybeUninit can be improved to generate higher quality code. + // + // Anyway, none of them works, thus we delay its initialization to the + // time when `reset` is called. Today Hafnium's implementation always + // call `reset` before using `ArchRegs`. But, the future is in shadow, + // we'd better refactor this, by making arch-dependent `new` methods. + unsafe { MaybeUninit::uninit().assume_init() } + } + /// Reset the register values other than the PC and argument which are set /// with `arch_regs_set_pc_arg()`. pub fn reset(&mut self, is_primary: bool, vm: &Vm, vcpu_id: cpu_id_t) { @@ -238,11 +267,19 @@ pub struct VCpuInner { /// ensures the scheduler can easily keep track of the vCPU state as /// transitions are indicated by the return code from the run call. pub state: VCpuStatus, - pub cpu: *mut Cpu, + pub cpu: *const Cpu, pub regs: ArchRegs, } impl VCpuInner { + pub fn new() -> Self { + Self { + state: VCpuStatus::Off, + cpu: ptr::null(), + regs: ArchRegs::new(), + } + } + /// Initialise the registers for the given vCPU and set the state to /// VCpuStatus::Ready. The caller must hold the vCPU execution lock while /// calling this. @@ -270,10 +307,24 @@ pub struct VCpu { pub interrupts: SpinLock, } +impl VCpu { + pub fn new(vm: *mut Vm) -> Self { + Self { + vm, + inner: SpinLock::new(VCpuInner::new()), + interrupts: SpinLock::new(Interrupts::new()), + } + } + + pub fn set_cpu(&mut self, cpu: *const Cpu) { + self.inner.get_mut().cpu = cpu; + } +} + /// Encapsulates a vCPU whose lock is held. #[repr(C)] pub struct VCpuExecutionLocked { - vcpu: *mut VCpu, + vcpu: *const VCpu, } impl Drop for VCpuExecutionLocked { @@ -295,12 +346,12 @@ impl Deref for VCpuExecutionLocked { impl VCpuExecutionLocked { #[inline] - pub unsafe fn from_raw(vcpu: *mut VCpu) -> Self { + pub unsafe fn from_raw(vcpu: *const VCpu) -> Self { Self { vcpu } } #[inline] - pub fn into_raw(self) -> *mut VCpu { + pub fn into_raw(self) -> *const VCpu { let ret = self.vcpu; mem::forget(self); ret @@ -331,98 +382,98 @@ pub struct Cpu { is_on: SpinLock, } -// unsafe impl Sync for Cpu {} - -/// The stack to be used by the CPUs. -/// TODO: alignas(2 * sizeof(uintreg_t)) -#[no_mangle] -static mut callstacks: MaybeUninit<[[u8; STACK_SIZE]; MAX_CPUS]> = MaybeUninit::uninit(); +impl Cpu { + pub fn new(id: cpu_id_t, stack_bottom: usize, is_on: bool) -> Self { + Self { + id, + stack_bottom: stack_bottom as *mut _, + is_on: SpinLock::new(is_on), + } + } +} -/// State of all supported CPUs. The stack of the first one is initialized. -/// Kernel loader writes booted CPU ID on `cpus.id` and initializes the CPU -/// stack by the address in `cpus.stack_bottom`. -/// (See src/arch/aarch64/hypervisor/plat_entry.S and cpu_entry.S.) -/// -/// Initializing static variables with pointers in Rust failed here. We left -/// the initialization code of `cpus` in `src/cpu.c`. -extern "C" { - #[no_mangle] - static mut cpus: MaybeUninit<[Cpu; MAX_CPUS]>; +pub struct CpuManager { + /// State of all supported CPUs. + cpus: ArrayVec<[Cpu; MAX_CPUS]>, } -static mut CPU_COUNT: u32 = 1; +impl CpuManager { + pub fn new( + cpu_ids: &[cpu_id_t], + boot_cpu_id: cpu_id_t, + stacks: &[[u8; STACK_SIZE]; MAX_CPUS], + ) -> Self { + let mut cpus: ArrayVec<[Cpu; MAX_CPUS]> = ArrayVec::new(); -#[no_mangle] -pub unsafe extern "C" fn cpu_init(_c: *mut Cpu) { - // TODO: Assumes that c is zeroed out already. -} + // Initialize boot CPU. + let boot_stack = stacks[0].as_ptr() as usize; + cpus.push(Cpu::new(boot_cpu_id, boot_stack + STACK_SIZE, true)); -#[no_mangle] -pub unsafe extern "C" fn cpu_module_init(cpu_ids: *mut cpu_id_t, count: usize) { - let mut j: u32; - let boot_cpu_id: cpu_id_t = cpus.get_ref()[0].id; - let mut found_boot_cpu: bool = false; - - arch_cpu_module_init(); - - CPU_COUNT = count as u32; - - // Initialize CPUs with the IDs from the configuration passed in. The - // CPUs after the boot CPU are initialized in reverse order. The boot - // CPU is initialized when it is found or in place of the last CPU if it - // is not found. - j = CPU_COUNT; - for i in 0..CPU_COUNT { - let c: *mut Cpu; - let id: cpu_id_t = *cpu_ids.offset(i as isize); - - if found_boot_cpu || id != boot_cpu_id { - j -= 1; - c = &mut cpus.get_mut()[j as usize]; - } else { - found_boot_cpu = true; - c = &mut cpus.get_mut()[0]; + // TODO(HfO2): Ask hafnium-discuss about zero or multiple boot CPU IDs + // and the reason why Hafnium initializes pCPUs in reverse order. If it + // has a special reason, fix this (#51.) + if cpu_ids.iter().filter(|id| boot_cpu_id == **id).count() != 1 { + panic!("`cpu_ids` contains zero or multiple boot CPU IDs.\n"); } - cpu_init(c); - (*c).id = id; - { - let callstacks_i = callstacks.get_mut()[i as usize].as_mut_ptr(); - (*c).stack_bottom = &mut *callstacks_i.add(STACK_SIZE) as *mut _ as _; + let cpu_ids = cpu_ids.iter().filter(|id| boot_cpu_id != **id); + let stacks = stacks.iter().skip(1); - // Note: referencing callstacks.get_mut()[i as usize][STACK_SIZE] directly - // causes 'index out of bounds' error on compile time. + for (cpu_id, stack) in cpu_ids.zip(stacks) { + cpus.push(Cpu::new( + *cpu_id, + stack.as_ptr() as usize + STACK_SIZE, + false, + )); } + + Self { cpus } + } + + pub unsafe fn index_of(&self, c: &Cpu) -> usize { + (c as *const Cpu).offset_from(self.cpus.as_ptr()) as usize + } + + pub unsafe fn cpu_on(&self, c: &Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { + let mut is_on = c.is_on.lock(); + let prev = *is_on; + *is_on = true; + + if !prev { + let vm = hafnium().vm_manager.get(HF_PRIMARY_VM_ID).unwrap(); + let vcpu = vm.vcpus.get(self.index_of(c)).unwrap(); + + vcpu.inner.lock().on(entry, arg); + } + + prev + } + + pub fn lookup(&self, id: cpu_id_t) -> Option<&Cpu> { + for cpu in self.cpus.iter() { + if cpu.id == id { + return Some(cpu); + } + } + + None } - if !found_boot_cpu { - // Boot CPU was initialized but with wrong ID. - dlog!("Boot CPU's ID not found in config."); - cpus.get_mut()[0].id = boot_cpu_id; + /// Temporal impl (WIP) + pub fn boot_cpu(&self) -> *mut Cpu { + self.cpus.get(0).unwrap() as &Cpu as *const _ as usize as *mut _ } } #[no_mangle] -pub unsafe extern "C" fn cpu_index(c: *mut Cpu) -> usize { - c.offset_from(cpus.get_ref().as_ptr()) as usize +pub unsafe extern "C" fn cpu_index(c: *const Cpu) -> usize { + hafnium().cpu_manager.index_of(&*c) } /// Turns CPU on and returns the previous state. #[no_mangle] pub unsafe extern "C" fn cpu_on(c: *mut Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { - let prev: bool; - let mut is_on = (*c).is_on.lock(); - prev = *is_on; - *is_on = true; - - if !prev { - let vm = vm_find(HF_PRIMARY_VM_ID); - let vcpu = vm_get_vcpu(vm, cpu_index(c) as u16); - - (*vcpu).inner.lock().on(entry, arg); - } - - prev + hafnium().cpu_manager.cpu_on(&*c, entry, arg) } /// Prepares the CPU for turning itself off. @@ -434,13 +485,11 @@ pub unsafe extern "C" fn cpu_off(c: *mut Cpu) { /// Searches for a CPU based on its id. #[no_mangle] pub unsafe extern "C" fn cpu_find(id: cpu_id_t) -> *mut Cpu { - for i in 0usize..CPU_COUNT as usize { - if cpus.get_ref()[i].id == id { - return &mut cpus.get_mut()[i]; - } - } - - ptr::null_mut() + hafnium() + .cpu_manager + .lookup(id) + .map(|cpu| cpu as *const _ as usize as *mut _) + .unwrap_or(ptr::null_mut()) } /// Locks the given vCPU and updates `locked` to hold the newly locked vCPU. @@ -471,13 +520,6 @@ pub unsafe extern "C" fn vcpu_unlock(locked: *mut VCpuExecutionLocked) { (*locked).vcpu = ptr::null_mut(); } -#[no_mangle] -pub unsafe extern "C" fn vcpu_init(vcpu: *mut VCpu, vm: *mut Vm) { - memset_s(vcpu as _, mem::size_of::(), 0, mem::size_of::()); - (*vcpu).vm = vm; - (*vcpu).inner.get_mut_unchecked().state = VCpuStatus::Off; -} - /// Initialise the registers for the given vCPU and set the state to /// VCpuStatus::Ready. The caller must hold the vCPU execution lock while /// calling this. @@ -511,13 +553,8 @@ pub unsafe extern "C" fn vcpu_get_vm(vcpu: *mut VCpu) -> *mut Vm { } #[no_mangle] -pub unsafe extern "C" fn vcpu_get_cpu(vcpu: *mut VCpu) -> *mut Cpu { - (*vcpu).inner.get_mut_unchecked().cpu -} - -#[no_mangle] -pub unsafe extern "C" fn vcpu_set_cpu(vcpu: *mut VCpu, cpu: *mut Cpu) { - (*vcpu).inner.get_mut_unchecked().cpu = cpu; +pub unsafe extern "C" fn vcpu_get_cpu(vcpu: *const VCpu) -> *mut Cpu { + (*vcpu).inner.get_mut_unchecked().cpu as usize as *mut _ } #[no_mangle] diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs new file mode 100644 index 000000000..5c0411c48 --- /dev/null +++ b/hfo2/src/fdt.rs @@ -0,0 +1,469 @@ +/* + * Copyright 2019 Sanguk Park. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::convert::{TryFrom, TryInto}; +use core::mem; +use core::ptr; +use core::slice; +use core::str; + +use crate::utils::*; + +use scopeguard::guard; + +/// TODO(HfO2): port this function into `std.rs` (#48.) +extern "C" { + fn strcmp(a: *const u8, b: *const u8) -> i64; +} + +#[derive(Clone)] +#[repr(C)] +pub struct FdtNode { + hdr: *const FdtHeader, + begin: *const u8, + end: *const u8, + strs: *const u8, +} + +#[repr(C)] +pub struct FdtHeader { + magic: u32, + totalsize: u32, + off_dt_struct: u32, + off_dt_strings: u32, + off_mem_rsvmap: u32, + version: u32, + last_comp_version: u32, + boot_cpuid_phys: u32, + size_dt_strings: u32, + size_dt_struct: u32, +} + +struct FdtReserveEntry { + address: u64, + size: u64, +} + +#[derive(PartialEq)] +enum FdtToken { + BeginNode = 1, + EndNode = 2, + Prop = 3, + Nop = 4, + End = 9, +} + +impl TryFrom for FdtToken { + type Error = (); + + fn try_from(value: u32) -> Result { + use FdtToken::*; + + match value { + 1 => Ok(BeginNode), + 2 => Ok(EndNode), + 3 => Ok(Prop), + 4 => Ok(Nop), + 9 => Ok(End), + _ => Err(()), + } + } +} + +struct FdtTokenizer { + cur: *const u8, + end: *const u8, + strs: *const u8, +} + +const FDT_VERSION: u32 = 17; +const FDT_MAGIC: u32 = 0xd00d_feed; +const FDT_TOKEN_ALIGNMENT: usize = mem::size_of::(); + +impl FdtTokenizer { + fn new(strs: *const u8, begin: *const u8, end: *const u8) -> Self { + Self { + cur: begin, + end, + strs, + } + } + + fn align(&mut self) { + self.cur = round_up(self.cur as usize, FDT_TOKEN_ALIGNMENT) as _; + } + + unsafe fn u32(&mut self) -> Option { + let next = self.cur.add(mem::size_of::()); + if next > self.end { + return None; + } + + let res = u32::from_be(*(self.cur as usize as *const u32)); + self.cur = next; + Some(res) + } + + unsafe fn token(&mut self) -> Option { + while let Some(v) = self.u32() { + let token = v.try_into().unwrap(); + if token != FdtToken::Nop { + return Some(token); + } + } + + None + } + + unsafe fn bytes(&mut self, size: usize) -> Option<*const u8> { + let next = self.cur.add(size); + + if next > self.end { + return None; + } + + let res = self.cur; + self.cur = next; + self.align(); + + Some(res) + } + + unsafe fn str(&mut self) -> Option<*const u8> { + let mut p = self.cur; + + while p < self.end { + if *p == 0 { + // Found the end of the string. + let res = self.cur; + self.cur = p.add(1); + self.align(); + return Some(res); + } + + p = p.add(1); + } + + None + } + + unsafe fn next_property(&mut self) -> Option<(*const u8, *const u8, u32)> { + let token = self.token()?; + + if token != FdtToken::Prop { + // Rewind so that caller will get the same token. + self.cur = self.cur.sub(mem::size_of::()); + return None; + } + + let mut this = guard(self, |this| { + // Move cursor to the end so that caller won't get any new tokens. + this.cur = this.end; + }); + + let size = this.u32()?; + let nameoff = this.u32()?; + let buf = this.bytes(size as usize)?; + + //TODO: Need to verify the strings. + let name = this.strs.add(nameoff as usize); + + mem::forget(this); + Some((name, buf, size)) + } + + unsafe fn next_subnode(&mut self) -> Option<*const u8> { + let token = self.token()?; + if token != FdtToken::BeginNode { + // Rewind so that caller will get the same token. + self.cur = self.cur.sub(mem::size_of::()); + return None; + } + + let name = unwrap_or!(self.str(), { + // Move cursor to the end so that caller won't get any new + // tokens. + self.cur = self.end; + return None; + }); + + Some(name) + } + + unsafe fn skip_properties(&mut self) { + while self.next_property().is_some() {} + } + + unsafe fn skip_node(&mut self) -> Option<()> { + let mut pending = 1; + self.skip_properties(); + + while pending != 0 { + while let Some(_) = self.next_subnode() { + self.skip_properties(); + pending += 1; + } + + let token = self.token()?; + if token != FdtToken::EndNode { + self.cur = self.end; + return None; + } + + pending -= 1; + } + + Some(()) + } +} + +impl FdtNode { + pub unsafe fn new_root(hdr: &FdtHeader) -> Option { + let begin = u32::from_be(hdr.off_dt_struct); + let size = u32::from_be(hdr.size_dt_struct); + + // Check the magic number before anything else. + if hdr.magic != u32::from_be(FDT_MAGIC) { + return None; + } + + // Check the version. + let max_ver = u32::from_be(hdr.version); + let min_ver = u32::from_be(hdr.last_comp_version); + if FDT_VERSION < min_ver || FDT_VERSION > max_ver { + return None; + } + + let hdr_ptr = hdr as *const _ as usize as *const u8; + + // TODO: Verify that it is all within the fdt. + // TODO: Verify strings as well. + Some(Self { + hdr: ptr::null(), + begin: hdr_ptr.add(begin as usize), + end: hdr_ptr.add((begin + size) as usize), + strs: hdr_ptr.add(u32::from_be(hdr.off_dt_strings) as usize), + }) + } + + pub unsafe fn read_property(&self, name: *const u8) -> Result<(*const u8, u32), ()> { + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); + while let Some((prop_name, buf, size)) = t.next_property() { + if strcmp(prop_name, name) == 0 { + return Ok((buf, size)); + } + } + + Err(()) + } + + pub unsafe fn first_child(&mut self) -> Option<*const u8> { + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); + + t.skip_properties(); + + let child_name = t.next_subnode()?; + self.begin = t.cur; + + Some(child_name) + } + + pub unsafe fn next_sibling(&mut self) -> Option<*const u8> { + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); + + t.skip_node()?; + let sibling_name = t.next_subnode()?; + self.begin = t.cur; + + Some(sibling_name) + } + + pub unsafe fn find_child(&mut self, child: *const u8) -> Option<()> { + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); + t.skip_properties(); + + while let Some(name) = t.next_subnode() { + if strcmp(name, child) == 0 { + self.begin = t.cur; + return Some(()); + } + + t.skip_node(); + } + + None + } +} + +impl FdtHeader { + pub unsafe fn dump(&self) { + unsafe fn asciz_to_utf8(ptr: *const u8) -> &'static str { + let len = (0..).find(|i| *ptr.add(*i) != 0).unwrap(); + let bytes = slice::from_raw_parts(ptr, len); + str::from_utf8_unchecked(bytes) + } + + // Traverse the whole thing. + let node = unwrap_or!(FdtNode::new_root(self), { + dlog!("FDT failed validation.\n"); + return; + }); + + let mut t = FdtTokenizer::new(node.strs, node.begin, node.end); + let mut depth = 0; + + loop { + while let Some(name) = t.next_subnode() { + dlog!( + "{:1$}New node: \"{2}\"\n", + "", + 2 * depth, + asciz_to_utf8(name) + ); + depth += 1; + while let Some((name, buf, size)) = t.next_property() { + dlog!( + "{:1$}property: \"{2}\" (", + "", + 2 * depth, + asciz_to_utf8(name) + ); + for i in 0..size as usize { + dlog!("{}{:02x}", if i == 0 { "" } else { " " }, *buf.add(i)); + } + dlog!(")\n"); + } + } + + if t.token().filter(|t| *t != FdtToken::EndNode).is_none() { + return; + } + + depth -= 1; + + if depth == 0 { + break; + } + } + + dlog!( + "fdt: off_mem_rsvmap={}\n", + u32::from_be(self.off_mem_rsvmap) + ); + + let mut entry = (self as *const _ as usize + u32::from_be(self.off_mem_rsvmap) as usize) + as *const FdtReserveEntry; + + while (*entry).address != 0 || (*entry).size != 0 { + dlog!( + "Entry: {:p} (0x{:x} bytes)\n", + u64::from_be((*entry).address) as *const u8, + u64::from_be((*entry).size) + ); + entry = entry.add(1); + } + } + + pub unsafe fn add_mem_reservation(&mut self, addr: u64, len: u64) { + // TODO: Clean this up. + let begin = + (self as *const _ as usize as *mut u8).add(u32::from_be(self.off_mem_rsvmap) as usize); + let e = begin as *mut FdtReserveEntry; + let old_size = (u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap)) as usize; + + self.totalsize = + (u32::from_be(self.totalsize) + mem::size_of::() as u32).to_be(); + self.off_dt_struct = + (u32::from_be(self.off_dt_struct) + mem::size_of::() as u32).to_be(); + self.off_dt_strings = + (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); + + ptr::copy( + begin, + begin.add(mem::size_of::()), + old_size, + ); + + (*e).address = addr.to_be(); + (*e).size = len.to_be(); + } + + pub fn total_size(&self) -> u32 { + u32::from_be(self.totalsize) + } +} + +#[no_mangle] +pub extern "C" fn fdt_header_size() -> usize { + mem::size_of::() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_total_size(hdr: *const FdtHeader) -> u32 { + (*hdr).total_size() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_dump(hdr: *mut FdtHeader) { + (*hdr).dump() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_root_node(node: *mut FdtNode, hdr: *const FdtHeader) -> bool { + let n = unwrap_or!(FdtNode::new_root(&*hdr), return false); + ptr::write(node, n); + true +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_child(node: *mut FdtNode, child: *const u8) -> bool { + (*node).find_child(child).is_some() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_first_child(node: *mut FdtNode, child_name: *mut *const u8) -> bool { + let name = unwrap_or!((*node).first_child(), return false); + ptr::write(child_name, name); + true +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_next_sibling( + node: *mut FdtNode, + sibling_name: *mut *const u8, +) -> bool { + let name = unwrap_or!((*node).next_sibling(), return false); + ptr::write(sibling_name, name); + true +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_read_property( + node: *mut FdtNode, + name: *const u8, + buf: *mut *const u8, + size: *mut u32, +) -> bool { + let (prop_buf, prop_size) = ok_or!((*node).read_property(name), return false); + *buf = prop_buf; + *size = prop_size; + true +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_add_mem_reserveation(hdr: *mut FdtHeader, addr: u64, len: u64) { + (*hdr).add_mem_reservation(addr, len) +} diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs new file mode 100644 index 000000000..3e6cc3c5c --- /dev/null +++ b/hfo2/src/fdt_handler.rs @@ -0,0 +1,497 @@ +/* + * Copyright 2019 Sanguk Park. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem; +use core::ptr::{self, NonNull}; + +use crate::addr::*; +use crate::arch::*; +use crate::boot_params::*; +use crate::fdt::*; +use crate::layout::*; +use crate::mm::*; +use crate::mpool::*; +use crate::page::*; +use crate::std::*; +use crate::types::*; + +use scopeguard::{guard, ScopeGuard}; + +unsafe fn convert_number(data: *const u8, size: u32) -> u64 { + match size { + 4 => u32::from_be(*(data as usize as *const u32)) as u64, + 8 => u64::from_be(*(data as usize as *const u64)), + _ => 0, + } +} + +impl FdtNode { + unsafe fn read_number(&mut self, name: *const u8) -> Result { + let (data, size) = self.read_property(name)?; + + match size { + 4 | 8 => Ok(convert_number(data, size)), + _ => Err(()), + } + } + + unsafe fn write_number(&mut self, name: *const u8, value: u64) -> Result<(), ()> { + let (data, size) = self.read_property(name)?; + + match size { + 4 => { + *(data as *mut u32) = u32::from_be(value as u32); + } + 8 => { + *(data as *mut u64) = u64::from_be(value); + } + _ => return Err(()), + } + + Ok(()) + } + + /// Finds the memory region where initrd is stored, and updates the fdt node + /// cursor to the node called "chosen". + unsafe fn find_initrd(&mut self, begin: &mut paddr_t, end: &mut paddr_t) -> bool { + if self.find_child("chosen\0".as_ptr()).is_none() { + dlog!("Unable to find 'chosen'\n"); + return false; + } + + let initrd_begin = ok_or!(self.read_number("linux,initrd-start\0".as_ptr()), { + dlog!("Unable to read linux,initrd-start\n"); + return false; + }); + + let initrd_end = ok_or!(self.read_number("linux,initrd-end\0".as_ptr()), { + dlog!("Unable to read linux,initrd-end\n"); + return false; + }); + + *begin = pa_init(initrd_begin as usize); + *end = pa_init(initrd_end as usize); + + true + } + + pub unsafe fn find_cpus(&self, cpu_ids: *mut cpu_id_t, cpu_count: &mut usize) -> Option<()> { + let mut node = self.clone(); + *cpu_count = 0; + + node.find_child("cpus\0".as_ptr()).or_else(|| { + dlog!("Unable to find 'cpus'\n"); + None + })?; + + let address_size = node + .read_number("#address-cells\0".as_ptr()) + .map(|size| size as usize * mem::size_of::()) + .unwrap_or(mem::size_of::()); + + node.first_child()?; + + // TODO(HfO2): this loop was do-while in C. Make an interator for this. + loop { + if node + .read_property("device_type\0".as_ptr()) + .ok() + .filter(|(_, size)| *size == "cpu\0".len() as u32) + .filter(|(data, _)| { + memcmp_rs( + *data as usize as *const _, + "cpu\0".as_ptr() as usize as *const _, + "cpu\0".len(), + ) == 0 + }) + .is_none() + { + if node.next_sibling().is_none() { + break; + } else { + continue; + } + } + + let (mut data, mut size) = + if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) { + (data, size) + } else { + if node.next_sibling().is_none() { + break; + } else { + continue; + } + }; + + // Get all entries for this CPU. + while size as usize >= address_size { + if *cpu_count >= MAX_CPUS { + dlog!("Found more than {} CPUs\n", MAX_CPUS); + return None; + } + + *cpu_ids.add(*cpu_count) = convert_number(data, address_size as u32) as cpu_id_t; + *cpu_count += 1; + + size -= address_size as u32; + data = data.add(address_size); + } + + if node.next_sibling().is_none() { + break; + } + } + + Some(()) + } + + pub unsafe fn find_memory_ranges(&self, p: &mut BootParams) -> Option<()> { + let mut node = self.clone(); + + // Get the sizes of memory range addresses and sizes. + let address_size = node + .read_number("#address-cells\0".as_ptr()) + .map(|size| size as usize * mem::size_of::()) + .unwrap_or(mem::size_of::()); + + let size_size = node + .read_number("#size-cells\0".as_ptr()) + .map(|size| size as usize * mem::size_of::()) + .unwrap_or(mem::size_of::()); + + let entry_size = address_size + size_size; + + // Look for nodes with the device_type set to "memory". + node.first_child()?; + let mut mem_range_index = 0; + + // TODO(HfO2): this loop was do-while in C. Make an interator for this. + loop { + if node + .read_property("device_type\0".as_ptr()) + .ok() + .filter(|(_, size)| *size as usize == "memory\0".len()) + .filter(|(data, _)| { + memcmp_rs( + *data as usize as *const _, + "memory\0".as_ptr() as usize as *const _, + "memory\0".len(), + ) == 0 + }) + .is_none() + { + if node.next_sibling().is_none() { + break; + } else { + continue; + } + } + let (mut data, mut size) = + if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) { + (data, size) + } else { + if node.next_sibling().is_none() { + break; + } else { + continue; + } + }; + + // Traverse all memory ranges within this node. + while size as usize >= entry_size { + let addr = convert_number(data, address_size as u32) as usize; + let len = convert_number(data.add(address_size), size_size as u32) as usize; + + if mem_range_index < MAX_MEM_RANGES { + p.mem_ranges[mem_range_index].begin = pa_init(addr); + p.mem_ranges[mem_range_index].end = pa_init(addr + len); + + mem_range_index += 1; + } else { + dlog!("Found memory range {} in FDT but only {} supported, ignoring additional range of size {}.\n", mem_range_index, MAX_MEM_RANGES, len); + } + + size -= entry_size as u32; + data = data.add(entry_size); + } + + if node.next_sibling().is_none() { + break; + } + } + + p.mem_ranges_count = mem_range_index; + Some(()) + } +} + +pub unsafe fn map( + stage1_ptable: &mut PageTable, + fdt_addr: paddr_t, + node: &mut FdtNode, + ppool: &mut MPool, +) -> Option> { + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + Mode::R, + ppool, + ) + .is_err() + { + dlog!("Unable to map FDT header.\n"); + return None; + } + + let mut stage1_ptable = guard(stage1_ptable, |ptable| { + let _ = ptable.unmap( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + ppool, + ); + }); + + let fdt = pa_addr(fdt_addr) as *mut FdtHeader; + + if let Some(root) = FdtNode::new_root(&*fdt) { + *node = root; + } else { + dlog!("FDT failed validation.\n"); + return None; + } + + // Map the rest of the fdt in. + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, (*fdt).total_size() as usize), + Mode::R, + ppool, + ) + .is_err() + { + dlog!("Unable to map full FDT.\n"); + return None; + } + + mem::forget(stage1_ptable); + NonNull::new(fdt) +} + +pub unsafe fn unmap( + stage1_ptable: &mut PageTable, + fdt: *mut FdtHeader, + ppool: &mut MPool, +) -> Result<(), ()> { + let fdt_addr = pa_from_va(va_from_ptr(fdt as usize as *const _)); + + stage1_ptable.unmap( + fdt_addr, + pa_add(fdt_addr, (*fdt).total_size() as usize), + ppool, + ) +} + +pub unsafe fn patch( + stage1_ptable: &mut PageTable, + fdt_addr: paddr_t, + p: &BootParamsUpdate, + ppool: &mut MPool, +) -> Result<(), ()> { + // Map the fdt header in. + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + Mode::R, + ppool, + ) + .is_err() + { + dlog!("Unable to map FDT header.\n"); + return Err(()); + } + + let mut stage1_ptable = guard(stage1_ptable, |ptable| { + let _ = ptable.unmap( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + ppool, + ); + }); + + let fdt = pa_addr(fdt_addr) as *mut FdtHeader; + + let mut node = FdtNode::new_root(&*fdt) + .or_else(|| { + dlog!("FDT failed validation.\n"); + None + }) + .ok_or(())?; + let total_size = (*fdt).total_size(); + + // Map the fdt (+ a page) in r/w mode in preparation for updating it. + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, total_size as usize + PAGE_SIZE), + Mode::R | Mode::W, + ppool, + ) + .is_err() + { + dlog!("Unable to map FDT in r/w mode.\n"); + return Err(()); + } + + let stage1_ptable = guard(ScopeGuard::into_inner(stage1_ptable), |ptable| { + if ptable + .unmap( + fdt_addr, + pa_add(fdt_addr, total_size as usize + PAGE_SIZE), + ppool, + ) + .is_err() + { + dlog!("Unable to unmap writable FDT.\n"); + } + }); + + if node.find_child("\0".as_ptr()).is_none() { + dlog!("Unable to find FDT root node.\n"); + return Err(()); + } + + if node.find_child("chosen\0".as_ptr()).is_none() { + dlog!("Unable to find 'chosen'\n"); + return Err(()); + } + + // Patch FDT to point to new ramdisk. + if node + .write_number( + "linux,initrd-start\0".as_ptr(), + pa_addr(p.initrd_begin) as u64, + ) + .is_err() + { + dlog!("Unable to write linux,initrd-start\n"); + return Err(()); + } + + if node + .write_number("linux,initrd-end\0".as_ptr(), pa_addr(p.initrd_end) as u64) + .is_err() + { + dlog!("Unable to write linux,initrd-end\n"); + return Err(()); + } + + // Patch FDT to reserve hypervisor memory so the primary VM doesn't try to + // use it. + (*fdt).add_mem_reservation( + pa_addr(layout_text_begin()) as u64, + pa_difference(layout_text_begin(), layout_text_end()) as u64, + ); + (*fdt).add_mem_reservation( + pa_addr(layout_rodata_begin()) as u64, + pa_difference(layout_rodata_begin(), layout_rodata_end()) as u64, + ); + (*fdt).add_mem_reservation( + pa_addr(layout_data_begin()) as u64, + pa_difference(layout_data_begin(), layout_data_end()) as u64, + ); + + // Patch FDT to reserve memory for secondary VMs. + for i in 0..p.reserved_ranges_count { + (*fdt).add_mem_reservation( + pa_addr(p.reserved_ranges[i].begin) as u64, + pa_addr(p.reserved_ranges[i].end) as u64 - pa_addr(p.reserved_ranges[i].begin) as u64, + ); + } + + let stage1_ptable = ScopeGuard::into_inner(stage1_ptable); + if stage1_ptable + .unmap( + fdt_addr, + pa_add(fdt_addr, (*fdt).total_size() as usize + PAGE_SIZE), + ppool, + ) + .is_err() + { + dlog!("Unable to unmap writable FDT.\n"); + return Err(()); + } + + Ok(()) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_map( + mut stage1_locked: mm_stage1_locked, + fdt_addr: paddr_t, + n: *mut FdtNode, + ppool: *mut MPool, +) -> *mut FdtHeader { + match map(&mut stage1_locked, fdt_addr, &mut *n, &mut *ppool) { + Some(ret) => ret.as_ptr(), + None => ptr::null_mut(), + } +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_unmap( + mut stage1_locked: mm_stage1_locked, + fdt: *mut FdtHeader, + ppool: *mut MPool, +) -> bool { + unmap(&mut stage1_locked, &mut *fdt, &mut *ppool).is_ok() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_cpus( + root: *const FdtNode, + cpu_ids: *mut cpu_id_t, + cpu_count: *mut usize, +) { + (*root).find_cpus(cpu_ids, &mut *cpu_count); +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_memory_ranges(root: *const FdtNode, p: *mut BootParams) { + (*root).find_memory_ranges(&mut *p); +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_initrd( + n: *mut FdtNode, + begin: *mut paddr_t, + end: *mut paddr_t, +) -> bool { + (*n).find_initrd(&mut *begin, &mut *end) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_patch( + mut stage1_locked: mm_stage1_locked, + fdt_addr: paddr_t, + p: *const BootParamsUpdate, + ppool: *mut MPool, +) -> bool { + patch(&mut stage1_locked, fdt_addr, &*p, &mut *ppool).is_ok() +} diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs new file mode 100644 index 000000000..a346b14c6 --- /dev/null +++ b/hfo2/src/init.rs @@ -0,0 +1,231 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem::MaybeUninit; +use core::ptr; + +use crate::addr::*; +use crate::arch::*; +use crate::boot_params::*; +use crate::cpu::*; +use crate::load::*; +use crate::memiter::*; +use crate::mm::*; +use crate::mpool::*; +use crate::page::*; +use crate::types::*; +use crate::vm::*; + +extern "C" { + fn plat_console_init(); + fn arch_one_time_init(); + fn dlog_enable_lock(); + + /// The stack to be used by the CPUs. + static callstacks: [[u8; STACK_SIZE]; MAX_CPUS]; + + /// A record for boot CPU. Its field `stack_bottom` is initialized. + /// Hafnium loader writes booted CPU ID on `cpus.id` and initializes the CPU + /// stack by the address in `cpus.stack_bottom`. + /// (See src/arch/aarch64/hypervisor/plat_entry.S and cpu_entry.S.) + /// + /// Initializing static variables with pointers in Rust failed here. We left + /// the initialization code of `cpus` in `src/cpu.c`. + static boot_cpu: Cpu; +} + +/// TODO(HfO2): Refactor api.rs and remove `pub` (#44.) +pub struct Hypervisor { + pub mpool: MPool, + pub memory_manager: MemoryManager, + pub cpu_manager: CpuManager, + pub vm_manager: VmManager, +} + +/// Note(HfO2): this variable was originally of type +/// MaybeUninit<[u8; mem::size_of::() * HEAP_PAGES]>, +/// but it was not aligned to PAGE_SIZE. +static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); + +static mut INITED: bool = false; + +/// A singleton collecting all managers in Hafnium. +/// +/// This is dependency-free; Typical solutions of mutable and shared static +/// objects delay their initialization. Considering concurrency, they often use +/// std::sync features to prevent racy initialization. But Hafnium is different. +/// +/// - The initialization is _always_ happened once in the specific time. +/// - During the time, no other thread is running; Hafnium runs as if it were +/// a single-thread program. +/// - After the initialization, Hafnium may make a non-exclusive reference of +/// singletons, but they have their own way for Hafnium to safely write them. +/// +/// Therefore, I do not use a safe wrapper for initialization such as +/// `std::sync::Once` and `lazy_static`. +/// +/// TODO(HfO2): This `pub` is required by mm_init, which is only used by unit +/// tests. Resolving #46 may help to remove the `pub`. +pub static mut HAFNIUM: MaybeUninit = MaybeUninit::uninit(); + +/// Performs one-time initialisation of the hypervisor. +#[no_mangle] +unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { + if INITED { + return c; + } + + // Make sure the console is initialised before calling dlog. + plat_console_init(); + + dlog!("Initialising hafnium\n"); + + arch_one_time_init(); + arch_cpu_module_init(); + + let mut ppool = MPool::new(); + ppool.free_pages(Pages::from_raw( + PTABLE_BUF.get_mut().as_mut_ptr(), + HEAP_PAGES, + )); + + let mm = MemoryManager::new(&ppool).expect("mm_init failed"); + + // We have to initialize memory manager early, because + // Stage2::invalidate_tlb refers global state. TODO(HfO2): Refactor the + // function to recieve the info from caller. Maybe MemoryManager should + // have a wrapper function for page table. + ptr::write(&mut HAFNIUM.get_mut().memory_manager, mm); + let mm = &HAFNIUM.get_ref().memory_manager; + + // Enable locks now that mm is initialised. + dlog_enable_lock(); + mpool_enable_locks(); + + let mut hypervisor_ptable = mm.hypervisor_ptable.lock(); + + let params = boot_params_get(&mut hypervisor_ptable, &mut ppool) + .expect("unable to retrieve boot params"); + + let cpum = CpuManager::new( + ¶ms.cpu_ids[..params.cpu_count], + boot_cpu.id, + &callstacks, + ); + + for i in 0..params.mem_ranges_count { + dlog!( + "Memory range: 0x{:x} - 0x{:x}\n", + pa_addr(params.mem_ranges[i].begin), + pa_addr(params.mem_ranges[i].end) - 1 + ); + } + + dlog!( + "Ramdisk range: 0x{:x} - 0x{:x}\n", + pa_addr(params.initrd_begin), + pa_addr(params.initrd_end) - 1 + ); + + // Map initrd in, and initialise cpio parser. + hypervisor_ptable + .identity_map(params.initrd_begin, params.initrd_end, Mode::R, &ppool) + .expect("unable to map initrd in"); + + let initrd = pa_addr(params.initrd_begin) as *mut _; + let cpio = MemIter::from_raw( + initrd, + pa_difference(params.initrd_begin, params.initrd_end), + ); + + ptr::write(&mut HAFNIUM.get_mut().vm_manager, VmManager::new()); + + // Load all VMs. + let primary_initrd = load_primary( + &mut HAFNIUM.get_mut().vm_manager, + &mut hypervisor_ptable, + &cpio, + params.kernel_arg, + &ppool, + ) + .expect("unable to load primary VM"); + + // load_secondary will add regions assigned to the secondary VMs from + // mem_ranges to reserved_ranges. + let mut update: BootParamsUpdate = BootParamsUpdate::new( + pa_from_va(va_from_ptr(primary_initrd.get_next() as usize as *const _)), + pa_from_va(va_from_ptr(primary_initrd.get_limit() as usize as *const _)), + ); + + load_secondary( + &mut HAFNIUM.get_mut().vm_manager, + &mut hypervisor_ptable, + &cpio, + ¶ms, + &mut update, + &mut ppool, + ) + .expect("unable to load secondary VMs"); + + // Prepare to run by updating bootparams as seen by primary VM. + boot_params_update(&mut hypervisor_ptable, &mut update, &mut ppool) + .expect("plat_update_boot_params failed"); + + hypervisor_ptable.defrag(&ppool); + + // Initialise HAFNIUM. + ptr::write(&mut HAFNIUM.get_mut().mpool, ppool); + ptr::write(&mut HAFNIUM.get_mut().cpu_manager, cpum); + + // Enable TLB invalidation for VM page table updates. + mm_vm_enable_invalidation(); + + dlog!("Hafnium initialisation completed\n"); + INITED = true; + + hafnium().cpu_manager.boot_cpu() + + // From now on, other pCPUs are on in order to run multiple vCPUs. Thus + // you may safely make readonly references to the Hafnium singleton, but + // may not modify the singleton without proper locking. +} + +pub fn hafnium() -> &'static Hypervisor { + unsafe { HAFNIUM.get_ref() } +} + +// The entry point of CPUs when they are turned on. It is supposed to initialise +// all state and return the first vCPU to run. +#[no_mangle] +pub unsafe extern "C" fn cpu_main(c: *const Cpu) -> *mut VCpu { + if !mm_cpu_init() { + panic!("mm_cpu_init failed"); + } + + let primary = hafnium().vm_manager.get(HF_PRIMARY_VM_ID).unwrap(); + let vcpu = primary.vcpus.get(cpu_index(&*c)).unwrap(); + let vm = vcpu.vm; + + // TODO(HfO2): vcpu needs to be borrowed exclusively, which is safe but + // discouraged. Move this code into one_time_init(). + let vcpu = vcpu as *const _ as usize as *mut VCpu; + (*vcpu).set_cpu(c); + + // Reset the registers to give a clean start for the primary's vCPU. + (*vcpu).inner.get_mut().regs.reset(true, &*vm, (*c).id); + + vcpu +} diff --git a/hfo2/src/layout.rs b/hfo2/src/layout.rs new file mode 100644 index 000000000..723491941 --- /dev/null +++ b/hfo2/src/layout.rs @@ -0,0 +1,38 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::addr::*; + +extern "C" { + pub fn layout_text_begin() -> paddr_t; + pub fn layout_text_end() -> paddr_t; + + pub fn layout_rodata_begin() -> paddr_t; + pub fn layout_rodata_end() -> paddr_t; + + pub fn layout_data_begin() -> paddr_t; + pub fn layout_data_end() -> paddr_t; + + pub fn layout_initrd_begin() -> paddr_t; + pub fn layout_initrd_end() -> paddr_t; + + pub fn layout_fdt_begin() -> paddr_t; + pub fn layout_fdt_end() -> paddr_t; + + pub fn layout_image_end() -> paddr_t; + + pub fn layout_primary_begin() -> paddr_t; +} diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index 709f5d22f..cd8369dfe 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -44,7 +44,13 @@ mod abi; mod addr; mod api; mod arch; +mod boot_params; mod cpu; +mod fdt; +mod fdt_handler; +mod init; +mod layout; +mod load; mod memiter; mod mm; mod mpool; diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs new file mode 100644 index 000000000..b3d09e69f --- /dev/null +++ b/hfo2/src/load.rs @@ -0,0 +1,329 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem; +use core::str; + +use crate::addr::*; +use crate::arch::*; +use crate::boot_params::*; +use crate::cpio::*; +use crate::cpu::*; +use crate::layout::*; +use crate::memiter::*; +use crate::mm::*; +use crate::mpool::*; +use crate::page::*; +use crate::std::*; +use crate::types::*; +use crate::utils::*; +use crate::vm::*; + +use arrayvec::ArrayVec; + +/// Copies data to an unmapped location by mapping it for write, copying the +/// data, then unmapping it. +/// +/// The data is written so that it is available to all cores with the cache +/// disabled. When switching to the partitions, the caching is initially +/// disabled so the data must be available without the cache. +unsafe fn copy_to_unmapped( + hypervisor_ptable: &mut PageTable, + to: paddr_t, + from: *const c_void, + size: usize, + ppool: &MPool, +) -> bool { + let to_end = pa_add(to, size); + + if hypervisor_ptable + .identity_map(to, to_end, Mode::W, ppool) + .is_err() + { + return false; + } + + memcpy_s(pa_addr(to) as *mut _, size, from, size); + arch_mm_write_back_dcache(pa_addr(to), size); + + hypervisor_ptable.unmap(to, to_end, ppool).unwrap(); + + true +} + +/// Loads the primary VM. +pub unsafe fn load_primary( + vm_manager: &mut VmManager, + hypervisor_ptable: &mut PageTable, + cpio: &MemIter, + kernel_arg: uintreg_t, + ppool: &MPool, +) -> Result { + let primary_begin = layout_primary_begin(); + + let it = unwrap_or!(find_file(&mut cpio.clone(), "vmlinuz\0".as_ptr()), { + dlog!("Unable to find vmlinuz\n"); + return Err(()); + }); + + dlog!( + "Copying primary to {:p}\n", + pa_addr(primary_begin) as *const u8 + ); + + if !copy_to_unmapped( + hypervisor_ptable, + primary_begin, + it.get_next() as usize as *mut _, + it.len(), + ppool, + ) { + dlog!("Unable to relocate kernel for primary vm.\n"); + return Err(()); + } + + let initrd = unwrap_or!(find_file(&mut cpio.clone(), "initrd.img\0".as_ptr()), { + dlog!("Unable to find initrd.img\n"); + return Err(()); + }); + + let vm = vm_manager + .new_vm(MAX_CPUS as spci_vcpu_count_t, ppool) + .ok_or_else(|| { + dlog!("Unable to initialise primary vm\n"); + (()) + })?; + + if vm.id != HF_PRIMARY_VM_ID { + dlog!("Primary vm was not given correct id\n"); + return Err(()); + } + + // Map the 1TB of memory. + // TODO: We should do a whitelist rather than blacklist. + if vm + .inner + .get_mut() + .ptable + .identity_map( + pa_init(0), + pa_init(1024usize * 1024 * 1024 * 1024), + Mode::R | Mode::W | Mode::X, + ppool, + ) + .is_err() + { + dlog!("Unable to initialise memory for primary vm\n"); + return Err(()); + } + + if !mm_vm_unmap_hypervisor(&mut (*vm).inner.get_mut_unchecked().ptable, ppool) { + dlog!("Unable to unmap hypervisor from primary vm\n"); + return Err(()); + } + + vm.vcpus[0] + .inner + .lock() // TODO(HfO2): We can safely use get_mut() here + .on(ipa_from_pa(primary_begin), kernel_arg); + + Ok(initrd) +} + +/// Try to find a memory range of the given size within the given ranges, and +/// remove it from them. Return true on success, or false if no large enough +/// contiguous range is found. +fn carve_out_mem_range( + mem_ranges: &mut [MemRange], + size_to_find: u64, +) -> Result<(paddr_t, paddr_t), ()> { + // TODO(b/116191358): Consider being cleverer about how we pack VMs + // together, with a non-greedy algorithm. + for mem_range in mem_ranges.iter_mut() { + if size_to_find <= pa_difference(mem_range.begin, mem_range.end) as u64 { + // This range is big enough, take some of it from the end and reduce + // its size accordingly. + let found_end = mem_range.end; + let found_begin = pa_init(pa_addr(mem_range.end) - size_to_find as usize); + mem_range.end = found_begin; + return Ok((found_begin, found_end)); + } + } + + Err(()) +} + +/// Given arrays of memory ranges before and after memory was removed for +/// secondary VMs, add the difference to the reserved ranges of the given +/// update. Return true on success, or false if there would be more than +/// MAX_MEM_RANGES reserved ranges after adding the new ones. +/// `before` and `after` must be arrays of exactly `mem_ranges_count` elements. +fn update_reserved_ranges( + update: &mut BootParamsUpdate, + before: &[MemRange], + after: &[MemRange], +) -> Result<(), ()> { + assert_eq!(before.len(), after.len()); + + for (before, after) in before.iter().zip(after.iter()) { + if pa_addr(after.begin) > pa_addr(before.begin) { + if update.reserved_ranges_count >= MAX_MEM_RANGES { + dlog!("Too many reserved ranges after loading secondary VMs.\n"); + return Err(()); + } + + update.reserved_ranges[update.reserved_ranges_count].begin = after.end; + update.reserved_ranges[update.reserved_ranges_count].end = before.end; + update.reserved_ranges_count += 1; + } + } + + Ok(()) +} + +/// Loads all secondary VMs into the memory ranges from the given params. +/// Memory reserved for the VMs is added to the `reserved_ranges` of `update`. +pub unsafe fn load_secondary( + vm_manager: &mut VmManager, + hypervisor_ptable: &mut PageTable, + cpio: &MemIter, + params: &BootParams, + update: &mut BootParamsUpdate, + ppool: &mut MPool, +) -> Result<(), ()> { + let mut mem_ranges_available: ArrayVec<[MemRange; MAX_MEM_RANGES]> = ArrayVec::new(); + // static_assert( + // sizeof(mem_ranges_available) == sizeof(params->mem_ranges), + // "mem_range arrays must be the same size for memcpy."); + + const_assert!(mem::size_of::() * MAX_MEM_RANGES < 500); + mem_ranges_available.set_len(MAX_MEM_RANGES); + mem_ranges_available.clone_from_slice(¶ms.mem_ranges); + mem_ranges_available.truncate(params.mem_ranges_count); + + let mut it = unwrap_or!(find_file(&mut cpio.clone(), "vms.txt\0".as_ptr()), { + dlog!("vms.txt is missing\n"); + return Ok(()); + }); + + // Round the last addresses down to the page size. + for mem_range in mem_ranges_available.iter_mut() { + mem_range.end = pa_init(round_down(pa_addr(mem_range.end), PAGE_SIZE)); + } + + loop { + // Note(HfO2): There is `while let (Some(x), Some(y)) = (...) {}` but it + // is not short-circuiting. + let mut mem = unwrap_or!(it.parse_uint(), break); + let cpu = unwrap_or!(it.parse_uint(), break); + let name = unwrap_or!(it.parse_str(), break); + let name_str = str::from_utf8_unchecked(name.as_slice()); + + dlog!("Loading {}\n", name_str); + + let kernel = unwrap_or!(find_file_memiter(&mut cpio.clone(), &name), { + dlog!("Unable to load kernel\n"); + continue; + }); + + // Round up to page size. + mem = (mem + PAGE_SIZE as u64 - 1) & !(PAGE_SIZE as u64 - 1); + + if mem < kernel.len() as u64 { + dlog!("Kernel is larger than available memory\n"); + continue; + } + + let (secondary_mem_begin, secondary_mem_end) = ok_or!( + carve_out_mem_range(&mut mem_ranges_available, mem as u64), + { + dlog!("Not enough memory ({} bytes)\n", mem); + continue; + } + ); + + if !copy_to_unmapped( + hypervisor_ptable, + secondary_mem_begin, + kernel.get_next() as usize as *const _, + kernel.len(), + ppool, + ) { + dlog!("Unable to copy kernel\n"); + continue; + } + + let primary = vm_manager.get_mut(HF_PRIMARY_VM_ID).unwrap(); + + // Deny the primary VM access to this memory. + if primary + .inner + .get_mut() + .ptable + .unmap(secondary_mem_begin, secondary_mem_end, ppool) + .is_err() + { + dlog!("Unable to unmap secondary VM from primary VM\n"); + return Err(()); + } + + let vm = unwrap_or!(vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool), { + dlog!("Unable to initialise VM\n"); + continue; + }); + + // Grant the VM access to the memory. + if vm + .inner + .get_mut() + .ptable + .identity_map( + secondary_mem_begin, + secondary_mem_end, + Mode::R | Mode::W | Mode::X, + ppool, + ) + .is_err() + { + dlog!("Unable to initialise memory\n"); + continue; + } + + dlog!( + "Loaded with {} vcpus, entry at 0x{:x}\n", + cpu, + pa_addr(secondary_mem_begin) + ); + + let secondary_entry = ipa_from_pa(secondary_mem_begin); + vcpu_secondary_reset_and_start( + &mut vm.vcpus[0], + secondary_entry, + pa_difference(secondary_mem_begin, secondary_mem_end) as uintreg_t, + ); + } + + // Add newly reserved areas to update params by looking at the difference + // between the available ranges from the original params and the updated + // mem_ranges_available. We assume that the number and order of available + // ranges is the same, i.e. we don't remove any ranges above only make them + // smaller. + update_reserved_ranges( + update, + ¶ms.mem_ranges[0..params.mem_ranges_count], + &mem_ranges_available, + ) +} diff --git a/hfo2/src/memiter.rs b/hfo2/src/memiter.rs index ed1a409ff..8af99bce7 100644 --- a/hfo2/src/memiter.rs +++ b/hfo2/src/memiter.rs @@ -15,6 +15,7 @@ */ use core::ptr; +use core::slice; use crate::std::*; use crate::types::*; @@ -145,6 +146,22 @@ impl MemIter { None } } + + pub fn get_next(&self) -> *const u8 { + self.next + } + + pub fn get_limit(&self) -> *const u8 { + self.limit + } + + pub unsafe fn as_slice(&self) -> &[u8] { + slice::from_raw_parts(self.next, self.limit.offset_from(self.next) as usize) + } + + pub fn len(&self) -> usize { + unsafe { self.limit.offset_from(self.next) as usize } + } } #[no_mangle] diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index 01243eae3..4c84899b0 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -36,6 +36,8 @@ use reduce::Reduce; use crate::addr::*; use crate::arch::*; +use crate::init::*; +use crate::layout::*; use crate::mpool::MPool; use crate::page::*; use crate::spinlock::{SpinLock, SpinLockGuard}; @@ -79,13 +81,6 @@ extern "C" { fn arch_mm_combine_table_entry_attrs(table_attrs: u64, block_attrs: u64) -> u64; fn plat_console_mm_init(stage1_locked: mm_stage1_locked, mpool: *const MPool); - - fn layout_text_begin() -> paddr_t; - fn layout_text_end() -> paddr_t; - fn layout_rodata_begin() -> paddr_t; - fn layout_rodata_end() -> paddr_t; - fn layout_data_begin() -> paddr_t; - fn layout_data_end() -> paddr_t; } bitflags! { @@ -168,13 +163,6 @@ type ptable_addr_t = uintvaddr_t; // For stage 2, the input is an intermediate physical addresses rather than a virtual address so: const_assert_eq!(addr_size_eq; mem::size_of::(), mem::size_of::()); -/// The hypervisor page table. -pub static HYPERVISOR_PAGE_TABLE: SpinLock> = - SpinLock::new(unsafe { PageTable::null() }); - -/// Is stage2 invalidation enabled? -pub static STAGE2_INVALIDATE: AtomicBool = AtomicBool::new(false); - /// Utility functions for address manipulation. mod addr { use super::ptable_addr_t; @@ -276,7 +264,11 @@ impl Stage for Stage2 { } fn invalidate_tlb(begin: usize, end: usize) { - if STAGE2_INVALIDATE.load(Ordering::Relaxed) { + if hafnium() + .memory_manager + .stage2_invalidate + .load(Ordering::Relaxed) + { unsafe { arch_mm_invalidate_stage2_range(ipa_init(begin), ipa_init(end)); } @@ -990,6 +982,12 @@ pub struct mm_stage1_locked { plock: usize, } +impl mm_stage1_locked { + pub fn from_ref(guard: &mut SpinLockGuard>) -> Self { + Self { plock: guard.raw() } + } +} + impl Deref for mm_stage1_locked { type Target = PageTable; @@ -1024,6 +1022,81 @@ impl<'s> Into>> for mm_stage1_locked { } } +pub struct MemoryManager { + /// The hypervisor page table. + pub hypervisor_ptable: SpinLock>, + + /// Is stage2 invalidation enabled? + pub stage2_invalidate: AtomicBool, +} + +impl MemoryManager { + pub fn new(mpool: &MPool) -> Option { + dlog!( + "text: {:#x} - {:#x}\n", + pa_addr(unsafe { layout_text_begin() }), + pa_addr(unsafe { layout_text_end() }) + ); + dlog!( + "rodata: {:#x} - {:#x}\n", + pa_addr(unsafe { layout_rodata_begin() }), + pa_addr(unsafe { layout_rodata_end() }) + ); + dlog!( + "data: {:#x} - {:#x}\n", + pa_addr(unsafe { layout_data_begin() }), + pa_addr(unsafe { layout_data_end() }) + ); + + let page_table = PageTable::new(mpool) + .map_err(|_| dlog!("Unable to allocate memory for page table.\n")) + .ok()?; + + // A fake lock. + // TODO(HfO2): IMO, mm_stage1_locked is better to hold a reference to + // PageTable rather than SpinLockGuard>, like + // other similar structures (VmLocked, VCpuExecutionLocked etc.) In that + // case we can delay creating a SpinLock here, and more directly show + // the meaning of unlocked but safe exclusive access of the page table. + let page_table = SpinLock::new(page_table); + let stage1_locked = mm_stage1_locked { + plock: &page_table as *const _ as usize, + }; + + unsafe { + // Let console driver map pages for itself. + plat_console_mm_init(stage1_locked, mpool); + + page_table + .get_mut_unchecked() + .identity_map(layout_text_begin(), layout_text_end(), Mode::X, mpool) + .unwrap(); + page_table + .get_mut_unchecked() + .identity_map(layout_rodata_begin(), layout_rodata_end(), Mode::R, mpool) + .unwrap(); + page_table + .get_mut_unchecked() + .identity_map( + layout_data_begin(), + layout_data_end(), + Mode::R | Mode::W, + mpool, + ) + .unwrap(); + + if !arch_mm_init(page_table.get_unchecked().root, true) { + return None; + } + } + + Some(Self { + hypervisor_ptable: page_table, + stage2_invalidate: AtomicBool::new(false), + }) + } +} + /// After calling this function, modifications to stage-2 page tables will use break-before-make and /// invalidate the TLB for the affected range. /// @@ -1032,7 +1105,10 @@ impl<'s> Into>> for mm_stage1_locked { /// This function should not be invoked concurrently with other memory management functions. #[no_mangle] pub unsafe extern "C" fn mm_vm_enable_invalidation() { - STAGE2_INVALIDATE.store(true, Ordering::Relaxed); + hafnium() + .memory_manager + .stage2_invalidate + .store(true, Ordering::Relaxed); } #[no_mangle] @@ -1097,17 +1173,17 @@ pub unsafe extern "C" fn mm_vm_unmap_hypervisor( // TODO: If we add pages dynamically, they must be included here too. let t = &mut *t; let mpool = &*mpool; - ok_or_return!( + ok_or!( t.unmap(layout_text_begin(), layout_text_end(), mpool), - false + return false ); - ok_or_return!( + ok_or!( t.unmap(layout_rodata_begin(), layout_rodata_end(), mpool), - false + return false ); - ok_or_return!( + ok_or!( t.unmap(layout_data_begin(), layout_data_end(), mpool), - false + return false ); true } @@ -1162,61 +1238,24 @@ pub unsafe extern "C" fn mm_unmap( stage1_locked.unmap(begin, end, mpool).is_ok() } +/// This function is only used in one unit test (fdt/find_memory_ranges.) +/// Unsafety doesn't really matter. #[no_mangle] pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { - dlog!( - "text: {:#x} - {:#x}\n", - pa_addr(layout_text_begin()), - pa_addr(layout_text_end()) - ); - dlog!( - "rodata: {:#x} - {:#x}\n", - pa_addr(layout_rodata_begin()), - pa_addr(layout_rodata_end()) - ); - dlog!( - "data: {:#x} - {:#x}\n", - pa_addr(layout_data_begin()), - pa_addr(layout_data_end()) - ); + let mm = unwrap_or!(MemoryManager::new(&*mpool), return false); + ptr::write(&mut HAFNIUM.get_mut().memory_manager, mm); - let mpool = &*mpool; - let page_table = - PageTable::new(mpool).map_err(|_| dlog!("Unable to allocate memory for page table.\n")); - let page_table = ok_or_return!(page_table, false); - let hypervisor_page_table = HYPERVISOR_PAGE_TABLE.get_mut_unchecked(); - ptr::write(hypervisor_page_table, page_table); - - // A fake lock. - let stage1_locked = mm_stage1_locked { - plock: &HYPERVISOR_PAGE_TABLE as *const _ as usize, - }; - - // Let console driver map pages for itself. - plat_console_mm_init(stage1_locked, mpool); - - hypervisor_page_table - .identity_map(layout_text_begin(), layout_text_end(), Mode::X, mpool) - .unwrap(); - hypervisor_page_table - .identity_map(layout_rodata_begin(), layout_rodata_end(), Mode::R, mpool) - .unwrap(); - hypervisor_page_table - .identity_map( - layout_data_begin(), - layout_data_end(), - Mode::R | Mode::W, - mpool, - ) - .unwrap(); - - arch_mm_init(hypervisor_page_table.root, true) + true } #[no_mangle] pub unsafe extern "C" fn mm_cpu_init() -> bool { - let ptable = HYPERVISOR_PAGE_TABLE.get_mut_unchecked().root; - arch_mm_init(ptable, false) + let raw_ptable = hafnium() + .memory_manager + .hypervisor_ptable + .get_mut_unchecked() + .root; + arch_mm_init(raw_ptable, false) } #[no_mangle] @@ -1227,7 +1266,8 @@ pub unsafe extern "C" fn mm_defrag(mut stage1_locked: mm_stage1_locked, mpool: * #[no_mangle] pub unsafe extern "C" fn mm_lock_stage1() -> mm_stage1_locked { - HYPERVISOR_PAGE_TABLE.lock().into() + let ptable = &hafnium().memory_manager.hypervisor_ptable; + ptable.lock().into() } #[no_mangle] diff --git a/hfo2/src/spci.rs b/hfo2/src/spci.rs index 08d55bea0..4563f2b35 100644 --- a/hfo2/src/spci.rs +++ b/hfo2/src/spci.rs @@ -132,7 +132,9 @@ impl SpciMessage { #[inline] pub fn get_architected_message_header(&self) -> &SpciArchitectedMessageHeader { #[allow(clippy::cast_ptr_alignment)] - unsafe { &*(self.payload.as_ptr() as *const _) } + unsafe { + &*(self.payload.as_ptr() as *const _) + } } } diff --git a/hfo2/src/spinlock.rs b/hfo2/src/spinlock.rs index 1bff66a7d..1a49f6f76 100644 --- a/hfo2/src/spinlock.rs +++ b/hfo2/src/spinlock.rs @@ -141,6 +141,12 @@ pub struct SpinLockGuard<'s, T> { unsafe impl<'s, T> Send for SpinLockGuard<'s, T> {} unsafe impl<'s, T: Send + Sync> Sync for SpinLockGuard<'s, T> {} +impl<'s, T> SpinLockGuard<'s, T> { + pub fn raw(&mut self) -> usize { + self.lock as *const _ as usize + } +} + impl<'s, T> Drop for SpinLockGuard<'s, T> { fn drop(&mut self) { self.lock.lock.unlock(); diff --git a/hfo2/src/types.rs b/hfo2/src/types.rs index f7ad1d7d3..d7db3dea0 100644 --- a/hfo2/src/types.rs +++ b/hfo2/src/types.rs @@ -57,11 +57,20 @@ pub const HF_INVALID_INTID: intid_t = 0xffff_ffff; /// The virtual interrupt ID used for the virtual timer. pub const HF_VIRTUAL_TIMER_INTID: intid_t = 3; -// These constants are originally from build scripts. Fortunately most -// testing environments have same conditions (MAX_CPUS=8, MAX_VMS=16.) And -// only one environment (host_fake) doesn't but it's for the unit test, so -// works fine under this settting (See //project/reference/BUILD.gn.) +// TODO(HfO2): These constants are originally from build scripts. (See +// //project/reference/BUILD.gn.) +pub const HEAP_PAGES: usize = 60; + +#[cfg(target_arch = "x86_64")] +pub const MAX_CPUS: usize = 4; + +#[cfg(target_arch = "aarch64")] pub const MAX_CPUS: usize = 8; + +#[cfg(target_arch = "x86_64")] +pub const MAX_VMS: usize = 6; + +#[cfg(target_arch = "aarch64")] pub const MAX_VMS: usize = 16; /// The ID of the primary VM which is responsible for scheduling. diff --git a/hfo2/src/utils.rs b/hfo2/src/utils.rs index 0a0c67294..5814e55b8 100644 --- a/hfo2/src/utils.rs +++ b/hfo2/src/utils.rs @@ -17,11 +17,21 @@ use core::sync::atomic::spin_loop_hint; #[macro_export] -macro_rules! ok_or_return { +macro_rules! ok_or { ($e:expr, $err:expr) => {{ match $e { Ok(r) => r, - Err(_) => return $err, + Err(_) => $err, + } + }}; +} + +#[macro_export] +macro_rules! unwrap_or { + ($e:expr, $err:expr) => {{ + match $e { + Some(r) => r, + None => $err, } }}; } diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index bf46afa48..94d32311f 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -14,8 +14,7 @@ * limitations under the License. */ -use core::mem; -use core::mem::MaybeUninit; +use core::mem::{self, MaybeUninit}; use core::ops::{Deref, DerefMut}; use core::ptr; use core::str; @@ -27,6 +26,7 @@ use scopeguard::guard; use crate::addr::*; use crate::arch::*; use crate::cpu::*; +use crate::init::*; use crate::list::*; use crate::mm::*; use crate::mpool::*; @@ -154,7 +154,9 @@ impl Mailbox { pa_recv_end: paddr_t, local_page_pool: &MPool, ) -> Result<(), ()> { - let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); + // TODO(HfO2): Acquring the singleton here is not recommended. Get the + // hypervisor ptable from callee (API module.) + let mut hypervisor_ptable = hafnium().memory_manager.hypervisor_ptable.lock(); let mut ptable = guard(hypervisor_ptable.deref_mut(), |_| ()); // Map the send page as read-only in the hypervisor address space. @@ -216,7 +218,7 @@ pub struct VmInner { impl VmInner { /// Initializes VmInner. - pub unsafe fn init(&mut self, vm: *mut Vm, ppool: &mut MPool) -> Result<(), ()> { + pub unsafe fn init(&mut self, vm: *mut Vm, ppool: &MPool) -> Result<(), ()> { self.mailbox.init(); if !mm_vm_init(&mut self.ptable, ppool) { @@ -461,7 +463,6 @@ impl VmInner { pub struct Vm { pub id: spci_vm_id_t, - pub vcpu_count: spci_vcpu_count_t, /// VCpus of this vm. /// Note: This field is regarded as a kind of mutable states of Vm, but is @@ -469,7 +470,7 @@ pub struct Vm { /// 1. Mutable inner fields are contained in VCpuState. /// 2. VCpuState has higher lock order than one of Vm. It is nonsense to /// lock VmInner to acquire VCpuState. - pub vcpus: [VCpu; MAX_CPUS], + pub vcpus: ArrayVec<[VCpu; MAX_CPUS]>, /// See api.c for the partial ordering on locks. pub inner: SpinLock, @@ -477,6 +478,38 @@ pub struct Vm { } impl Vm { + pub fn init( + &mut self, + id: spci_vm_id_t, + vcpu_count: spci_vcpu_count_t, + ppool: &MPool, + ) -> Result<(), ()> { + self.id = id; + // self.vcpus = ArrayVec::new(); + // TODO(HfO2): Using former one will allocate large temporal stack to + // store a ArrayVec<[VCpu; MAX_CPUS]>. Maybe MIR-only RLIBs or denoting + // ArrayVec::new as inline will solve this problem. + // (See https://github.com/rust-lang/rust/issues/38913) + unsafe { + self.vcpus = MaybeUninit::uninit().assume_init(); + self.vcpus.set_len(0); + } + self.aborting = AtomicBool::new(false); + unsafe { + let self_ptr = self as *mut _; + self.inner.get_mut().init(self_ptr, ppool)?; + + for _ in 0..vcpu_count { + // self.vcpus.push(VCpu::new(self_ptr)); + // TODO(HfO2): Using former one will allocate large temporal + // stack to store a VCpu. Maybe MIR-only RLIBs or denoting + // ArrayVec::push as inline will solve this problem. + self.vcpus.push_unchecked(VCpu::new(self_ptr)); + } + } + Ok(()) + } + /// Returns the root address of the page table of this VM. It is safe not to /// lock `self.inner` because the value of `ptable.as_raw()` doesn't change /// after `ptable` is initialized. Of course, actual page table may vary @@ -539,62 +572,63 @@ impl VmLocked { } } -static mut VMS: MaybeUninit<[Vm; MAX_VMS]> = MaybeUninit::uninit(); -static mut VM_COUNT: spci_vm_count_t = 0; - -#[no_mangle] -pub unsafe extern "C" fn vm_init( - vcpu_count: spci_vcpu_count_t, - ppool: *mut MPool, - new_vm: *mut *mut Vm, -) -> bool { - let vm: *mut Vm; +pub struct VmManager { + vms: ArrayVec<[Vm; MAX_VMS]>, +} - if VM_COUNT as usize >= MAX_VMS { - return false; +impl VmManager { + pub fn new() -> Self { + Self { + vms: ArrayVec::new(), + } } - vm = &mut VMS.get_mut()[VM_COUNT as usize]; + pub fn new_vm(&mut self, vcpu_count: spci_vcpu_count_t, ppool: &MPool) -> Option<&mut Vm> { + if self.vms.is_full() { + return None; + } - memset_s( - vm as usize as _, - mem::size_of::(), - 0, - mem::size_of::(), - ); + let id = self.vms.len(); + let vm = unsafe { self.vms.get_unchecked_mut(id) }; - (*vm).id = VM_COUNT; - (*vm).vcpu_count = vcpu_count; - (*vm).aborting = AtomicBool::new(false); - let result = (*vm).inner.get_mut_unchecked().init(vm, &mut *ppool); - if result.is_err() { - return false; - } + vm.init(id as u16, vcpu_count, ppool).ok()?; + + unsafe { + self.vms.set_len(id + 1); + } - // Do basic initialization of vcpus. - for i in 0..vcpu_count { - vcpu_init(vm_get_vcpu(vm, i), vm); + Some(&mut self.vms[id]) } - VM_COUNT += 1; - *new_vm = vm; + pub fn get(&self, id: spci_vm_id_t) -> Option<&Vm> { + self.vms.get(id as usize) + } - true + pub fn get_mut(&mut self, id: spci_vm_id_t) -> Option<&mut Vm> { + self.vms.get_mut(id as usize) + } } +/// This function is only used by unit test (fdt/find_memory_ranges.) #[no_mangle] -pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { - VM_COUNT +pub unsafe extern "C" fn vm_init( + vcpu_count: spci_vcpu_count_t, + ppool: *mut MPool, + new_vm: *mut *mut Vm, +) -> bool { + let vmm = &hafnium().vm_manager as *const _ as usize as *mut VmManager; + match (*vmm).new_vm(vcpu_count, &*ppool) { + Some(vm) => { + *new_vm = vm as *mut _; + true + } + None => false, + } } #[no_mangle] -pub unsafe extern "C" fn vm_find(id: spci_vm_id_t) -> *mut Vm { - // Ensure the VM is initialized. - if id >= VM_COUNT { - return ptr::null_mut(); - } - - &mut VMS.get_mut()[id as usize] +pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { + hafnium().vm_manager.vms.len() as _ } /// Locks the given VM and updates `locked` to hold the newly locked vm. @@ -616,7 +650,7 @@ pub unsafe extern "C" fn vm_unlock(locked: *mut VmLocked) { /// This assumes the index is valid, i.e. less than vm->vcpu_count. #[no_mangle] pub unsafe extern "C" fn vm_get_vcpu(vm: *mut Vm, vcpu_index: spci_vcpu_index_t) -> *mut VCpu { - assert!(vcpu_index < (*vm).vcpu_count); + assert!((vcpu_index as usize) < (*vm).vcpus.len()); &mut (*vm).vcpus[vcpu_index as usize] } @@ -637,5 +671,5 @@ pub unsafe extern "C" fn vm_get_arch(vm: *mut Vm) -> *mut ArchVm { #[no_mangle] pub unsafe extern "C" fn vm_get_vcpu_count(vm: *const Vm) -> spci_vcpu_count_t { - (*vm).vcpu_count + (*vm).vcpus.len() as _ } diff --git a/inc/hf/api.h b/inc/hf/api.h index 9ba9ca534..855ad07f9 100644 --- a/inc/hf/api.h +++ b/inc/hf/api.h @@ -22,7 +22,6 @@ #include "vmapi/hf/call.h" -void api_init(struct mpool *ppool); spci_vm_id_t api_vm_get_id(const struct vcpu *current); spci_vm_count_t api_vm_get_count(void); spci_vcpu_count_t api_vcpu_get_count(spci_vm_id_t vm_id, diff --git a/inc/hf/cpu.h b/inc/hf/cpu.h index 2f75a78c1..b6b6b6455 100644 --- a/inc/hf/cpu.h +++ b/inc/hf/cpu.h @@ -97,8 +97,6 @@ struct cpu { bool is_on; }; -void cpu_module_init(const cpu_id_t *cpu_ids, size_t count); - size_t cpu_index(struct cpu *c); bool cpu_on(struct cpu *c, ipaddr_t entry, uintreg_t arg); void cpu_off(struct cpu *c); @@ -107,14 +105,12 @@ struct cpu *cpu_find(cpu_id_t id); struct vcpu_execution_locked vcpu_lock(struct vcpu *vcpu); bool vcpu_try_lock(struct vcpu *vcpu, struct vcpu_execution_locked *locked); void vcpu_unlock(struct vcpu_execution_locked *locked); -void vcpu_init(struct vcpu *vcpu, struct vm *vm); void vcpu_on(struct vcpu_execution_locked vcpu, ipaddr_t entry, uintreg_t arg); spci_vcpu_index_t vcpu_index(const struct vcpu *vcpu); struct arch_regs *vcpu_get_regs(struct vcpu *vcpu); const struct arch_regs *vcpu_get_regs_const(const struct vcpu *vcpu); struct vm *vcpu_get_vm(struct vcpu *vcpu); struct cpu *vcpu_get_cpu(struct vcpu *vcpu); -void vcpu_set_cpu(struct vcpu *vcpu, struct cpu *cpu); struct interrupts *vcpu_get_interrupts(struct vcpu *vcpu); bool vcpu_is_off(struct vcpu_execution_locked vcpu); bool vcpu_secondary_reset_and_start(struct vcpu *vcpu, ipaddr_t entry, diff --git a/inc/hf/fdt_handler.h b/inc/hf/fdt_handler.h index 49d242e5b..8adee755b 100644 --- a/inc/hf/fdt_handler.h +++ b/inc/hf/fdt_handler.h @@ -33,4 +33,4 @@ bool fdt_find_initrd(struct fdt_node *n, paddr_t *begin, paddr_t *end); /** Apply an update to the FDT. */ bool fdt_patch(struct mm_stage1_locked stage1_locked, paddr_t fdt_addr, - struct boot_params_update *p, struct mpool *ppool); + const struct boot_params_update *p, struct mpool *ppool); diff --git a/inc/hf/vm.h b/inc/hf/vm.h index c65830739..d11894fd0 100644 --- a/inc/hf/vm.h +++ b/inc/hf/vm.h @@ -94,7 +94,6 @@ struct vm_locked { bool vm_init(spci_vcpu_count_t vcpu_count, struct mpool *ppool, struct vm **new_vm); spci_vm_count_t vm_get_count(void); -struct vm *vm_find(spci_vm_id_t id); struct vm_locked vm_lock(struct vm *vm); void vm_unlock(struct vm_locked *locked); struct vcpu *vm_get_vcpu(struct vm *vm, spci_vcpu_index_t vcpu_index); diff --git a/inc/vmapi/hf/abi.h b/inc/vmapi/hf/abi.h index 538a42e5b..4682f9c21 100644 --- a/inc/vmapi/hf/abi.h +++ b/inc/vmapi/hf/abi.h @@ -121,32 +121,6 @@ enum hf_share { HF_MEMORY_SHARE, }; -/** - * Encode an hf_vcpu_run_return struct in the 64-bit packing ABI. - */ -static inline uint64_t hf_vcpu_run_return_encode(struct hf_vcpu_run_return res) -{ - uint64_t ret = res.code & 0xff; - - switch (res.code) { - case HF_VCPU_RUN_WAKE_UP: - ret |= (uint64_t)res.wake_up.vm_id << 32; - ret |= (uint64_t)res.wake_up.vcpu << 16; - break; - case HF_VCPU_RUN_MESSAGE: - ret |= res.message.vm_id << 8; - break; - case HF_VCPU_RUN_WAIT_FOR_INTERRUPT: - case HF_VCPU_RUN_WAIT_FOR_MESSAGE: - ret |= res.sleep.ns << 8; - break; - default: - break; - } - - return ret; -} - /** * Decode an hf_vcpu_run_return struct from the 64-bit packing ABI. */ diff --git a/kokoro/ubuntu/test.sh b/kokoro/ubuntu/test.sh index c29a1fe45..492452a2a 100755 --- a/kokoro/ubuntu/test.sh +++ b/kokoro/ubuntu/test.sh @@ -61,6 +61,8 @@ $TIMEOUT 30s $OUT/host_fake_clang/unit_tests \ --gtest_output="xml:$OUT/kokoro_log/unit_tests/sponge_log.xml" \ | tee $OUT/kokoro_log/unit_tests/sponge_log.log +cargo test --manifest-path=hfo2/Cargo.toml + $HFTEST arch_test $HFTEST hafnium --initrd test/vmapi/gicv3/gicv3_test $HFTEST hafnium --initrd test/vmapi/primary_only/primary_only_test diff --git a/src/BUILD.gn b/src/BUILD.gn index 11369d43b..a2cf268b0 100644 --- a/src/BUILD.gn +++ b/src/BUILD.gn @@ -29,8 +29,6 @@ hypervisor("hafnium") { # src targets will merge! source_set("src_not_testable_yet") { sources = [ - "load.c", - "main.c", "plat.c", ] libs = ["//hfo2/target/aarch64-hfo2/release/libhfo2.a"] @@ -85,31 +83,10 @@ source_set("dlog") { } source_set("fdt_handler") { - sources = [ - "fdt_handler.c", - ] - deps = [ - ":fdt", - ] - - if (is_debug) { - deps += [ ":dlog" ] - } } # Flattened Device Tree (FDT) utilities. source_set("fdt") { - sources = [ - "fdt.c", - ] - - deps = [ - ":std", - ] - - if (is_debug) { - deps += [ ":dlog" ] - } } source_set("memiter") { diff --git a/src/abi_test.cc b/src/abi_test.cc index 204472d4f..d5b3968bf 100644 --- a/src/abi_test.cc +++ b/src/abi_test.cc @@ -24,28 +24,6 @@ namespace { using ::testing::Eq; -/** - * Simulate an uninitialized hf_vcpu_run_return so it can be detected if any - * uninitialized fields make their way into the encoded form which would - * indicate a data leak. - */ -struct hf_vcpu_run_return dirty_vcpu_run_return() -{ - struct hf_vcpu_run_return res; - memset(&res, 0xc5, sizeof(res)); - return res; -} - -/** - * Encode a preempted response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_preempted) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_PREEMPTED; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0)); -} - /** * Decode a preempted response ignoring the irrelevant bits. */ @@ -56,16 +34,6 @@ TEST(abi, hf_vcpu_run_return_decode_preempted) EXPECT_THAT(res.code, Eq(HF_VCPU_RUN_PREEMPTED)); } -/** - * Encode a yield response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_yield) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_YIELD; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(1)); -} - /** * Decode a yield response ignoring the irrelevant bits. */ @@ -76,29 +44,6 @@ TEST(abi, hf_vcpu_run_return_decode_yield) EXPECT_THAT(res.code, Eq(HF_VCPU_RUN_YIELD)); } -/** - * Encode wait-for-interrupt response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_interrupt) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT; - res.sleep.ns = HF_SLEEP_INDEFINITE; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0xffffffffffffff02)); -} - -/** - * Encoding wait-for-interrupt response with too large sleep duration will drop - * the top octet. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_interrupt_sleep_too_long) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT; - res.sleep.ns = 0xcc22888888888888; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x2288888888888802)); -} - /** * Decode a wait-for-interrupt response ignoring the irrelevant bits. */ @@ -110,29 +55,6 @@ TEST(abi, hf_vcpu_run_return_decode_wait_for_interrupt) EXPECT_THAT(res.sleep.ns, Eq(0x1234abcdbadb01)); } -/** - * Encode wait-for-message response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_message) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_MESSAGE; - res.sleep.ns = HF_SLEEP_INDEFINITE; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0xffffffffffffff03)); -} - -/** - * Encoding wait-for-message response with too large sleep duration will drop - * the top octet. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_message_sleep_too_long) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_MESSAGE; - res.sleep.ns = 0xaa99777777777777; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x9977777777777703)); -} - /** * Decode a wait-for-message response ignoring the irrelevant bits. */ @@ -144,18 +66,6 @@ TEST(abi, hf_vcpu_run_return_decode_wait_for_message) EXPECT_THAT(res.sleep.ns, Eq(0x12347654badb01)); } -/** - * Encode wake up response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_wake_up) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAKE_UP; - res.wake_up.vm_id = 0x1234; - res.wake_up.vcpu = 0xabcd; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x1234abcd0004)); -} - /** * Decode a wake up response ignoring the irrelevant bits. */ @@ -168,17 +78,6 @@ TEST(abi, hf_vcpu_run_return_decode_wake_up) EXPECT_THAT(res.wake_up.vcpu, Eq(0xf00d)); } -/** - * Encode message response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_message) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_MESSAGE; - res.message.vm_id = 0xf007; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x0000000000f00705)); -} - /** * Decode a wake up response ignoring the irrelevant bits. */ @@ -190,16 +89,6 @@ TEST(abi, hf_vcpu_run_return_decode_message) EXPECT_THAT(res.message.vm_id, Eq(0x9162)); } -/** - * Encode a 'notify waiters' response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_notify_waiters) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_NOTIFY_WAITERS; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(6)); -} - /** * Decode a 'notify waiters' response ignoring the irrelevant bits. */ @@ -210,16 +99,6 @@ TEST(abi, hf_vcpu_run_return_decode_notify_waiters) EXPECT_THAT(res.code, Eq(HF_VCPU_RUN_NOTIFY_WAITERS)); } -/** - * Encode an aborted response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_aborted) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_ABORTED; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(7)); -} - /** * Decode an aborted response ignoring the irrelevant bits. */ diff --git a/src/arch/aarch64/hypervisor/cpu_entry.S b/src/arch/aarch64/hypervisor/cpu_entry.S index 301a1a10d..316fe0435 100644 --- a/src/arch/aarch64/hypervisor/cpu_entry.S +++ b/src/arch/aarch64/hypervisor/cpu_entry.S @@ -34,6 +34,9 @@ cpu_entry: add x30, x30, :lo12:vector_table_el2 msr vbar_el2, x30 + /* Initialize Hafnium. */ + bl one_time_init + /* Call into C code, x0 holds the cpu pointer. */ bl cpu_main diff --git a/src/arch/aarch64/hypervisor/hypervisor_entry.S b/src/arch/aarch64/hypervisor/hypervisor_entry.S index e797074ee..8cdb0647d 100644 --- a/src/arch/aarch64/hypervisor/hypervisor_entry.S +++ b/src/arch/aarch64/hypervisor/hypervisor_entry.S @@ -23,8 +23,8 @@ image_entry: bl plat_entry /* Get pointer to first cpu. */ - adrp x0, cpus - add x0, x0, :lo12:cpus + adrp x0, boot_cpu + add x0, x0, :lo12:boot_cpu /* Set the ID of this cpu from the affinity bits of mpidr. */ mrs x30, mpidr_el1 @@ -33,4 +33,12 @@ image_entry: orr x30, x29, x30 str x30, [x0, CPU_ID] - b cpu_entry + /* Call into C code, x0 holds the cpu pointer. */ + bl cpu_entry + + /* Run the vcpu returned by cpu_main. */ + bl vcpu_restore_all_and_run + + /* Loop forever waiting for interrupts. */ +0: wfi + b 0b diff --git a/src/cpu.c b/src/cpu.c index fc0ab62b3..b832f9798 100644 --- a/src/cpu.c +++ b/src/cpu.c @@ -32,12 +32,12 @@ #define STACK_SIZE PAGE_SIZE /* The stack to be used by the CPUs. */ -extern char callstacks[MAX_CPUS][STACK_SIZE]; - -/* State of all supported CPUs. The stack of the first one is initialized. */ -struct cpu cpus[MAX_CPUS] = { - { - .is_on = 1, - .stack_bottom = &callstacks[0][STACK_SIZE], - }, +alignas(2 * sizeof(uintreg_t)) char callstacks[MAX_CPUS][STACK_SIZE]; + +/** + * A temporal variable for one-time booting sequence. The booting CPU will + * refer this. + */ +struct cpu boot_cpu = { + .stack_bottom = &callstacks[0][STACK_SIZE], }; diff --git a/src/fdt.c b/src/fdt.c deleted file mode 100644 index de54189f4..000000000 --- a/src/fdt.c +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hf/fdt.h" - -#include - -#include "hf/dlog.h" -#include "hf/std.h" - -struct fdt_header { - uint32_t magic; - uint32_t totalsize; - uint32_t off_dt_struct; - uint32_t off_dt_strings; - uint32_t off_mem_rsvmap; - uint32_t version; - uint32_t last_comp_version; - uint32_t boot_cpuid_phys; - uint32_t size_dt_strings; - uint32_t size_dt_struct; -}; - -struct fdt_reserve_entry { - uint64_t address; - uint64_t size; -}; - -enum fdt_token { - FDT_BEGIN_NODE = 1, - FDT_END_NODE = 2, - FDT_PROP = 3, - FDT_NOP = 4, - FDT_END = 9, -}; - -struct fdt_tokenizer { - const char *cur; - const char *end; - const char *strs; -}; - -#define FDT_VERSION 17 -#define FDT_MAGIC 0xd00dfeed - -static void fdt_tokenizer_init(struct fdt_tokenizer *t, const char *strs, - const char *begin, const char *end) -{ - t->strs = strs; - t->cur = begin; - t->end = end; -} - -static void fdt_tokenizer_align(struct fdt_tokenizer *t) -{ - t->cur = (char *)align_up(t->cur, 4); -} - -static bool fdt_tokenizer_uint32(struct fdt_tokenizer *t, uint32_t *res) -{ - const char *next = t->cur + sizeof(*res); - - if (next > t->end) { - return false; - } - - *res = be32toh(*(uint32_t *)t->cur); - t->cur = next; - - return true; -} - -static bool fdt_tokenizer_token(struct fdt_tokenizer *t, uint32_t *res) -{ - uint32_t v; - - while (fdt_tokenizer_uint32(t, &v)) { - if (v != FDT_NOP) { - *res = v; - return true; - } - } - return false; -} - -static bool fdt_tokenizer_bytes(struct fdt_tokenizer *t, const char **res, - size_t size) -{ - const char *next = t->cur + size; - - if (next > t->end) { - return false; - } - - *res = t->cur; - t->cur = next; - fdt_tokenizer_align(t); - - return true; -} - -static bool fdt_tokenizer_str(struct fdt_tokenizer *t, const char **res) -{ - const char *p; - - for (p = t->cur; p < t->end; p++) { - if (!*p) { - /* Found the end of the string. */ - *res = t->cur; - t->cur = p + 1; - fdt_tokenizer_align(t); - return true; - } - } - - return false; -} - -bool fdt_root_node(struct fdt_node *node, const struct fdt_header *hdr) -{ - uint32_t max_ver; - uint32_t min_ver; - uint32_t begin = be32toh(hdr->off_dt_struct); - uint32_t size = be32toh(hdr->size_dt_struct); - - memset_s(node, sizeof(*node), 0, sizeof(*node)); - - /* Check the magic number before anything else. */ - if (hdr->magic != be32toh(FDT_MAGIC)) { - return false; - } - - /* Check the version. */ - max_ver = be32toh(hdr->version); - min_ver = be32toh(hdr->last_comp_version); - if (FDT_VERSION < min_ver || FDT_VERSION > max_ver) { - return false; - } - - /* TODO: Verify that it is all within the fdt. */ - node->begin = (const char *)hdr + begin; - node->end = node->begin + size; - - /* TODO: Verify strings as well. */ - node->strs = (char *)hdr + be32toh(hdr->off_dt_strings); - - return true; -} - -static bool fdt_next_property(struct fdt_tokenizer *t, const char **name, - const char **buf, uint32_t *size) -{ - uint32_t token; - uint32_t nameoff; - - if (!fdt_tokenizer_token(t, &token)) { - return false; - } - - if (token != FDT_PROP) { - /* Rewind so that caller will get the same token. */ - t->cur -= sizeof(uint32_t); - return false; - } - - if (!fdt_tokenizer_uint32(t, size) || - !fdt_tokenizer_uint32(t, &nameoff) || - !fdt_tokenizer_bytes(t, buf, *size)) { - /* - * Move cursor to the end so that caller won't get any new - * tokens. - */ - t->cur = t->end; - return false; - } - - /* TODO: Need to verify the strings. */ - *name = t->strs + nameoff; - - return true; -} - -static bool fdt_next_subnode(struct fdt_tokenizer *t, const char **name) -{ - uint32_t token; - - if (!fdt_tokenizer_token(t, &token)) { - return false; - } - - if (token != FDT_BEGIN_NODE) { - /* Rewind so that caller will get the same token. */ - t->cur -= sizeof(uint32_t); - return false; - } - - if (!fdt_tokenizer_str(t, name)) { - /* - * Move cursor to the end so that caller won't get any new - * tokens. - */ - t->cur = t->end; - return false; - } - - return true; -} - -static void fdt_skip_properties(struct fdt_tokenizer *t) -{ - const char *name; - const char *buf; - uint32_t size; - - while (fdt_next_property(t, &name, &buf, &size)) { - /* do nothing */ - } -} - -static bool fdt_skip_node(struct fdt_tokenizer *t) -{ - const char *name; - uint32_t token; - size_t pending = 1; - - fdt_skip_properties(t); - - do { - while (fdt_next_subnode(t, &name)) { - fdt_skip_properties(t); - pending++; - } - - if (!fdt_tokenizer_token(t, &token)) { - return false; - } - - if (token != FDT_END_NODE) { - t->cur = t->end; - return false; - } - - pending--; - } while (pending); - - return true; -} - -bool fdt_read_property(const struct fdt_node *node, const char *name, - const char **buf, uint32_t *size) -{ - struct fdt_tokenizer t; - const char *prop_name; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - while (fdt_next_property(&t, &prop_name, buf, size)) { - if (!strcmp(prop_name, name)) { - return true; - } - } - - return false; -} - -bool fdt_first_child(struct fdt_node *node, const char **child_name) -{ - struct fdt_tokenizer t; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - fdt_skip_properties(&t); - - if (!fdt_next_subnode(&t, child_name)) { - return false; - } - - node->begin = t.cur; - - return true; -} - -bool fdt_next_sibling(struct fdt_node *node, const char **sibling_name) -{ - struct fdt_tokenizer t; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - if (!fdt_skip_node(&t)) { - return false; - } - - if (!fdt_next_subnode(&t, sibling_name)) { - return false; - } - - node->begin = t.cur; - - return true; -} - -bool fdt_find_child(struct fdt_node *node, const char *child) -{ - struct fdt_tokenizer t; - const char *name; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - fdt_skip_properties(&t); - - while (fdt_next_subnode(&t, &name)) { - if (!strcmp(name, child)) { - node->begin = t.cur; - return true; - } - - fdt_skip_node(&t); - } - - return false; -} - -void fdt_dump(struct fdt_header *hdr) -{ - uint32_t token; - size_t depth = 0; - const char *name; - struct fdt_tokenizer t; - struct fdt_node node; - - /* Traverse the whole thing. */ - if (!fdt_root_node(&node, hdr)) { - dlog("FDT failed validation.\n"); - return; - } - - fdt_tokenizer_init(&t, node.strs, node.begin, node.end); - - do { - while (fdt_next_subnode(&t, &name)) { - const char *buf; - uint32_t size; - - dlog("%*sNew node: \"%s\"\n", 2 * depth, "", name); - depth++; - while (fdt_next_property(&t, &name, &buf, &size)) { - uint32_t i; - - dlog("%*sproperty: \"%s\" (", 2 * depth, "", - name); - for (i = 0; i < size; i++) { - dlog("%s%02x", i == 0 ? "" : " ", - buf[i]); - } - dlog(")\n"); - } - } - - if (!fdt_tokenizer_token(&t, &token)) { - return; - } - - if (token != FDT_END_NODE) { - return; - } - - depth--; - } while (depth); - - dlog("fdt: off_mem_rsvmap=%u\n", be32toh(hdr->off_mem_rsvmap)); - { - struct fdt_reserve_entry *e = - (struct fdt_reserve_entry - *)((uintptr_t)hdr + - be32toh(hdr->off_mem_rsvmap)); - while (e->address || e->size) { - dlog("Entry: %p (0x%x bytes)\n", be64toh(e->address), - be64toh(e->size)); - e++; - } - } -} - -void fdt_add_mem_reservation(struct fdt_header *hdr, uint64_t addr, - uint64_t len) -{ - /* TODO: Clean this up. */ - uint8_t *begin = (uint8_t *)hdr + be32toh(hdr->off_mem_rsvmap); - struct fdt_reserve_entry *e = (struct fdt_reserve_entry *)begin; - size_t old_size = - be32toh(hdr->totalsize) - be32toh(hdr->off_mem_rsvmap); - - hdr->totalsize = htobe32(be32toh(hdr->totalsize) + - sizeof(struct fdt_reserve_entry)); - hdr->off_dt_struct = htobe32(be32toh(hdr->off_dt_struct) + - sizeof(struct fdt_reserve_entry)); - hdr->off_dt_strings = htobe32(be32toh(hdr->off_dt_strings) + - sizeof(struct fdt_reserve_entry)); - memmove_s(begin + sizeof(struct fdt_reserve_entry), old_size, begin, - old_size); - e->address = htobe64(addr); - e->size = htobe64(len); -} - -size_t fdt_header_size(void) -{ - return sizeof(struct fdt_header); -} - -uint32_t fdt_total_size(struct fdt_header *hdr) -{ - return be32toh(hdr->totalsize); -} diff --git a/src/fdt_handler.c b/src/fdt_handler.c deleted file mode 100644 index db8c0ef6e..000000000 --- a/src/fdt_handler.c +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hf/fdt_handler.h" - -#include "hf/boot_params.h" -#include "hf/cpu.h" -#include "hf/dlog.h" -#include "hf/fdt.h" -#include "hf/layout.h" -#include "hf/mm.h" -#include "hf/std.h" - -static uint64_t convert_number(const char *data, uint32_t size) -{ - union { - volatile uint64_t v; - char a[8]; - } t; - - switch (size) { - case sizeof(uint32_t): - return be32toh(*(uint32_t *)data); - case sizeof(uint64_t): - memcpy_s(t.a, sizeof(t.a), data, sizeof(uint64_t)); - return be64toh(t.v); - default: - return 0; - } -} - -static bool fdt_read_number(const struct fdt_node *node, const char *name, - uint64_t *value) -{ - const char *data; - uint32_t size; - - if (!fdt_read_property(node, name, &data, &size)) { - return false; - } - - switch (size) { - case sizeof(uint32_t): - case sizeof(uint64_t): - *value = convert_number(data, size); - break; - - default: - return false; - } - - return true; -} - -static bool fdt_write_number(struct fdt_node *node, const char *name, - uint64_t value) -{ - const char *data; - uint32_t size; - union { - volatile uint64_t v; - char a[8]; - } t; - - if (!fdt_read_property(node, name, &data, &size)) { - return false; - } - - switch (size) { - case sizeof(uint32_t): - *(uint32_t *)data = be32toh(value); - break; - - case sizeof(uint64_t): - t.v = be64toh(value); - memcpy_s((void *)data, size, t.a, sizeof(uint64_t)); - break; - - default: - return false; - } - - return true; -} - -/** - * Finds the memory region where initrd is stored, and updates the fdt node - * cursor to the node called "chosen". - */ -bool fdt_find_initrd(struct fdt_node *n, paddr_t *begin, paddr_t *end) -{ - uint64_t initrd_begin; - uint64_t initrd_end; - - if (!fdt_find_child(n, "chosen")) { - dlog("Unable to find 'chosen'\n"); - return false; - } - - if (!fdt_read_number(n, "linux,initrd-start", &initrd_begin)) { - dlog("Unable to read linux,initrd-start\n"); - return false; - } - - if (!fdt_read_number(n, "linux,initrd-end", &initrd_end)) { - dlog("Unable to read linux,initrd-end\n"); - return false; - } - - *begin = pa_init(initrd_begin); - *end = pa_init(initrd_end); - - return true; -} - -void fdt_find_cpus(const struct fdt_node *root, cpu_id_t *cpu_ids, - size_t *cpu_count) -{ - struct fdt_node n = *root; - const char *name; - uint64_t address_size; - - *cpu_count = 0; - - if (!fdt_find_child(&n, "cpus")) { - dlog("Unable to find 'cpus'\n"); - return; - } - - if (fdt_read_number(&n, "#address-cells", &address_size)) { - address_size *= sizeof(uint32_t); - } else { - address_size = sizeof(uint32_t); - } - - if (!fdt_first_child(&n, &name)) { - return; - } - - do { - const char *data; - uint32_t size; - - if (!fdt_read_property(&n, "device_type", &data, &size) || - size != sizeof("cpu") || - memcmp(data, "cpu", sizeof("cpu")) != 0 || - !fdt_read_property(&n, "reg", &data, &size)) { - continue; - } - - /* Get all entries for this CPU. */ - while (size >= address_size) { - if (*cpu_count >= MAX_CPUS) { - dlog("Found more than %d CPUs\n", MAX_CPUS); - return; - } - - cpu_ids[(*cpu_count)++] = - convert_number(data, address_size); - - size -= address_size; - data += address_size; - } - } while (fdt_next_sibling(&n, &name)); -} - -void fdt_find_memory_ranges(const struct fdt_node *root, struct boot_params *p) -{ - struct fdt_node n = *root; - const char *name; - uint64_t address_size; - uint64_t size_size; - uint64_t entry_size; - size_t mem_range_index = 0; - - /* Get the sizes of memory range addresses and sizes. */ - if (fdt_read_number(&n, "#address-cells", &address_size)) { - address_size *= sizeof(uint32_t); - } else { - address_size = sizeof(uint32_t); - } - - if (fdt_read_number(&n, "#size-cells", &size_size)) { - size_size *= sizeof(uint32_t); - } else { - size_size = sizeof(uint32_t); - } - - entry_size = address_size + size_size; - - /* Look for nodes with the device_type set to "memory". */ - if (!fdt_first_child(&n, &name)) { - return; - } - - do { - const char *data; - uint32_t size; - - if (!fdt_read_property(&n, "device_type", &data, &size) || - size != sizeof("memory") || - memcmp(data, "memory", sizeof("memory")) != 0 || - !fdt_read_property(&n, "reg", &data, &size)) { - continue; - } - - /* Traverse all memory ranges within this node. */ - while (size >= entry_size) { - uintpaddr_t addr = convert_number(data, address_size); - size_t len = - convert_number(data + address_size, size_size); - - if (mem_range_index < MAX_MEM_RANGES) { - p->mem_ranges[mem_range_index].begin = - pa_init(addr); - p->mem_ranges[mem_range_index].end = - pa_init(addr + len); - ++mem_range_index; - } else { - dlog("Found memory range %u in FDT but only " - "%u supported, ignoring additional range " - "of size %u.\n", - mem_range_index, MAX_MEM_RANGES, len); - } - - size -= entry_size; - data += entry_size; - } - } while (fdt_next_sibling(&n, &name)); - p->mem_ranges_count = mem_range_index; - - /* TODO: Check for "reserved-memory" nodes. */ -} - -struct fdt_header *fdt_map(struct mm_stage1_locked stage1_locked, - paddr_t fdt_addr, struct fdt_node *n, - struct mpool *ppool) -{ - struct fdt_header *fdt; - - /* Map the fdt header in. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_header_size()), MM_MODE_R, - ppool); - if (!fdt) { - dlog("Unable to map FDT header.\n"); - return NULL; - } - - if (!fdt_root_node(n, fdt)) { - dlog("FDT failed validation.\n"); - goto fail; - } - - /* Map the rest of the fdt in. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt)), MM_MODE_R, - ppool); - if (!fdt) { - dlog("Unable to map full FDT.\n"); - goto fail; - } - - return fdt; - -fail: - mm_unmap(stage1_locked, fdt_addr, pa_add(fdt_addr, fdt_header_size()), - ppool); - return NULL; -} - -bool fdt_unmap(struct mm_stage1_locked stage1_locked, struct fdt_header *fdt, - struct mpool *ppool) -{ - paddr_t fdt_addr = pa_from_va(va_from_ptr(fdt)); - - return mm_unmap(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt)), ppool); -} - -bool fdt_patch(struct mm_stage1_locked stage1_locked, paddr_t fdt_addr, - struct boot_params_update *p, struct mpool *ppool) -{ - struct fdt_header *fdt; - struct fdt_node n; - bool ret = false; - size_t i; - - /* Map the fdt header in. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_header_size()), MM_MODE_R, - ppool); - if (!fdt) { - dlog("Unable to map FDT header.\n"); - return false; - } - - if (!fdt_root_node(&n, fdt)) { - dlog("FDT failed validation.\n"); - goto err_unmap_fdt_header; - } - - /* Map the fdt (+ a page) in r/w mode in preparation for updating it. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt) + PAGE_SIZE), - MM_MODE_R | MM_MODE_W, ppool); - if (!fdt) { - dlog("Unable to map FDT in r/w mode.\n"); - goto err_unmap_fdt_header; - } - - if (!fdt_find_child(&n, "")) { - dlog("Unable to find FDT root node.\n"); - goto out_unmap_fdt; - } - - if (!fdt_find_child(&n, "chosen")) { - dlog("Unable to find 'chosen'\n"); - goto out_unmap_fdt; - } - - /* Patch FDT to point to new ramdisk. */ - if (!fdt_write_number(&n, "linux,initrd-start", - pa_addr(p->initrd_begin))) { - dlog("Unable to write linux,initrd-start\n"); - goto out_unmap_fdt; - } - - if (!fdt_write_number(&n, "linux,initrd-end", pa_addr(p->initrd_end))) { - dlog("Unable to write linux,initrd-end\n"); - goto out_unmap_fdt; - } - - /* - * Patch FDT to reserve hypervisor memory so the primary VM doesn't try - * to use it. - */ - fdt_add_mem_reservation( - fdt, pa_addr(layout_text_begin()), - pa_difference(layout_text_begin(), layout_text_end())); - fdt_add_mem_reservation( - fdt, pa_addr(layout_rodata_begin()), - pa_difference(layout_rodata_begin(), layout_rodata_end())); - fdt_add_mem_reservation( - fdt, pa_addr(layout_data_begin()), - pa_difference(layout_data_begin(), layout_data_end())); - - /* Patch FDT to reserve memory for secondary VMs. */ - for (i = 0; i < p->reserved_ranges_count; ++i) { - fdt_add_mem_reservation( - fdt, pa_addr(p->reserved_ranges[i].begin), - pa_addr(p->reserved_ranges[i].end) - - pa_addr(p->reserved_ranges[i].begin)); - } - - ret = true; - -out_unmap_fdt: - /* Unmap FDT. */ - if (!mm_unmap(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt) + PAGE_SIZE), - ppool)) { - dlog("Unable to unmap writable FDT.\n"); - return false; - } - return ret; - -err_unmap_fdt_header: - mm_unmap(stage1_locked, fdt_addr, pa_add(fdt_addr, fdt_header_size()), - ppool); - return false; -} diff --git a/src/load.c b/src/load.c deleted file mode 100644 index 57167d4a8..000000000 --- a/src/load.c +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hf/load.h" - -#include - -#include "hf/api.h" -#include "hf/boot_params.h" -#include "hf/dlog.h" -#include "hf/layout.h" -#include "hf/memiter.h" -#include "hf/mm.h" -#include "hf/plat/console.h" -#include "hf/static_assert.h" -#include "hf/std.h" -#include "hf/vm.h" - -#include "vmapi/hf/call.h" - -/** - * Copies data to an unmapped location by mapping it for write, copying the - * data, then unmapping it. - * - * The data is written so that it is available to all cores with the cache - * disabled. When switching to the partitions, the caching is initially disabled - * so the data must be available without the cache. - */ -static bool copy_to_unmapped(struct mm_stage1_locked stage1_locked, paddr_t to, - const void *from, size_t size, struct mpool *ppool) -{ - paddr_t to_end = pa_add(to, size); - void *ptr; - - ptr = mm_identity_map(stage1_locked, to, to_end, MM_MODE_W, ppool); - if (!ptr) { - return false; - } - - memcpy_s(ptr, size, from, size); - arch_mm_write_back_dcache(ptr, size); - - mm_unmap(stage1_locked, to, to_end, ppool); - - return true; -} - -/** - * Loads the primary VM. - */ -bool load_primary(struct mm_stage1_locked stage1_locked, - const struct memiter *cpio, uintreg_t kernel_arg, - struct memiter *initrd, struct mpool *ppool) -{ - struct memiter it; - paddr_t primary_begin = layout_primary_begin(); - - if (!cpio_find_file(cpio, "vmlinuz", &it)) { - dlog("Unable to find vmlinuz\n"); - return false; - } - - dlog("Copying primary to %p\n", pa_addr(primary_begin)); - if (!copy_to_unmapped(stage1_locked, primary_begin, it.next, - it.limit - it.next, ppool)) { - dlog("Unable to relocate kernel for primary vm.\n"); - return false; - } - - if (!cpio_find_file(cpio, "initrd.img", initrd)) { - dlog("Unable to find initrd.img\n"); - return false; - } - - { - struct vm *vm; - struct vcpu_execution_locked vcpu_execution_locked; - - if (!vm_init(MAX_CPUS, ppool, &vm)) { - dlog("Unable to initialise primary vm\n"); - return false; - } - - if (vm_get_id(vm) != HF_PRIMARY_VM_ID) { - dlog("Primary vm was not given correct id\n"); - return false; - } - - /* Map the 1TB of memory. */ - /* TODO: We should do a whitelist rather than a blacklist. */ - if (!mm_vm_identity_map( - vm_get_ptable(vm), pa_init(0), - pa_init(UINT64_C(1024) * 1024 * 1024 * 1024), - MM_MODE_R | MM_MODE_W | MM_MODE_X, NULL, ppool)) { - dlog("Unable to initialise memory for primary vm\n"); - return false; - } - - if (!mm_vm_unmap_hypervisor(vm_get_ptable(vm), ppool)) { - dlog("Unable to unmap hypervisor from primary vm\n"); - return false; - } - - vcpu_execution_locked = vcpu_lock(vm_get_vcpu(vm, 0)); - vcpu_on(vcpu_execution_locked, ipa_from_pa(primary_begin), kernel_arg); - vcpu_unlock(&vcpu_execution_locked); - } - - return true; -} - -/** - * Try to find a memory range of the given size within the given ranges, and - * remove it from them. Return true on success, or false if no large enough - * contiguous range is found. - */ -static bool carve_out_mem_range(struct mem_range *mem_ranges, - size_t mem_ranges_count, uint64_t size_to_find, - paddr_t *found_begin, paddr_t *found_end) -{ - size_t i; - - /* - * TODO(b/116191358): Consider being cleverer about how we pack VMs - * together, with a non-greedy algorithm. - */ - for (i = 0; i < mem_ranges_count; ++i) { - if (size_to_find <= - pa_difference(mem_ranges[i].begin, mem_ranges[i].end)) { - /* - * This range is big enough, take some of it from the - * end and reduce its size accordingly. - */ - *found_end = mem_ranges[i].end; - *found_begin = pa_init(pa_addr(mem_ranges[i].end) - - size_to_find); - mem_ranges[i].end = *found_begin; - return true; - } - } - return false; -} - -/** - * Given arrays of memory ranges before and after memory was removed for - * secondary VMs, add the difference to the reserved ranges of the given update. - * Return true on success, or false if there would be more than MAX_MEM_RANGES - * reserved ranges after adding the new ones. - * `before` and `after` must be arrays of exactly `mem_ranges_count` elements. - */ -static bool update_reserved_ranges(struct boot_params_update *update, - const struct mem_range *before, - const struct mem_range *after, - size_t mem_ranges_count) -{ - size_t i; - - for (i = 0; i < mem_ranges_count; ++i) { - if (pa_addr(after[i].begin) > pa_addr(before[i].begin)) { - if (update->reserved_ranges_count >= MAX_MEM_RANGES) { - dlog("Too many reserved ranges after loading " - "secondary VMs.\n"); - return false; - } - update->reserved_ranges[update->reserved_ranges_count] - .begin = before[i].begin; - update->reserved_ranges[update->reserved_ranges_count] - .end = after[i].begin; - update->reserved_ranges_count++; - } - if (pa_addr(after[i].end) < pa_addr(before[i].end)) { - if (update->reserved_ranges_count >= MAX_MEM_RANGES) { - dlog("Too many reserved ranges after loading " - "secondary VMs.\n"); - return false; - } - update->reserved_ranges[update->reserved_ranges_count] - .begin = after[i].end; - update->reserved_ranges[update->reserved_ranges_count] - .end = before[i].end; - update->reserved_ranges_count++; - } - } - - return true; -} - -/** - * Loads all secondary VMs into the memory ranges from the given params. - * Memory reserved for the VMs is added to the `reserved_ranges` of `update`. - */ -bool load_secondary(struct mm_stage1_locked stage1_locked, - const struct memiter *cpio, - const struct boot_params *params, - struct boot_params_update *update, struct mpool *ppool) -{ - struct vm *primary; - struct memiter it; - struct memiter name; - uint64_t mem; - uint64_t cpu; - struct mem_range mem_ranges_available[MAX_MEM_RANGES]; - size_t i; - - static_assert( - sizeof(mem_ranges_available) == sizeof(params->mem_ranges), - "mem_range arrays must be the same size for memcpy."); - static_assert(sizeof(mem_ranges_available) < 500, - "This will use too much stack, either make " - "MAX_MEM_RANGES smaller or change this."); - memcpy_s(mem_ranges_available, sizeof(mem_ranges_available), - params->mem_ranges, sizeof(params->mem_ranges)); - - primary = vm_find(HF_PRIMARY_VM_ID); - - if (!cpio_find_file(cpio, "vms.txt", &it)) { - dlog("vms.txt is missing\n"); - return true; - } - - /* Round the last addresses down to the page size. */ - for (i = 0; i < params->mem_ranges_count; ++i) { - mem_ranges_available[i].end = pa_init(align_down( - pa_addr(mem_ranges_available[i].end), PAGE_SIZE)); - } - - while (memiter_parse_uint(&it, &mem) && memiter_parse_uint(&it, &cpu) && - memiter_parse_str(&it, &name)) { - struct memiter kernel; - paddr_t secondary_mem_begin; - paddr_t secondary_mem_end; - ipaddr_t secondary_entry; - const char *p; - struct vm *vm; - struct vcpu *vcpu; - - dlog("Loading "); - for (p = name.next; p != name.limit; ++p) { - dlog("%c", *p); - } - dlog("\n"); - - if (!cpio_find_file_memiter(cpio, &name, &kernel)) { - dlog("Unable to load kernel\n"); - continue; - } - - /* Round up to page size. */ - mem = (mem + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - - if (mem < kernel.limit - kernel.next) { - dlog("Kernel is larger than available memory\n"); - continue; - } - - if (!carve_out_mem_range( - mem_ranges_available, params->mem_ranges_count, mem, - &secondary_mem_begin, &secondary_mem_end)) { - dlog("Not enough memory (%u bytes)\n", mem); - continue; - } - - if (!copy_to_unmapped(stage1_locked, secondary_mem_begin, - kernel.next, kernel.limit - kernel.next, - ppool)) { - dlog("Unable to copy kernel\n"); - continue; - } - - if (!vm_init(cpu, ppool, &vm)) { - dlog("Unable to initialise VM\n"); - continue; - } - - /* Grant the VM access to the memory. */ - if (!mm_vm_identity_map(vm_get_ptable(vm), secondary_mem_begin, - secondary_mem_end, - MM_MODE_R | MM_MODE_W | MM_MODE_X, - &secondary_entry, ppool)) { - dlog("Unable to initialise memory\n"); - continue; - } - - /* Deny the primary VM access to this memory. */ - if (!mm_vm_unmap(vm_get_ptable(primary), secondary_mem_begin, - secondary_mem_end, ppool)) { - dlog("Unable to unmap secondary VM from primary VM\n"); - return false; - } - - dlog("Loaded with %u vcpus, entry at 0x%x\n", cpu, - pa_addr(secondary_mem_begin)); - - vcpu = vm_get_vcpu(vm, 0); - vcpu_secondary_reset_and_start( - vcpu, secondary_entry, - pa_difference(secondary_mem_begin, secondary_mem_end)); - } - - /* - * Add newly reserved areas to update params by looking at the - * difference between the available ranges from the original params and - * the updated mem_ranges_available. We assume that the number and order - * of available ranges is the same, i.e. we don't remove any ranges - * above only make them smaller. - */ - return update_reserved_ranges(update, params->mem_ranges, - mem_ranges_available, - params->mem_ranges_count); -} diff --git a/src/main.c b/src/main.c deleted file mode 100644 index 6695292cc..000000000 --- a/src/main.c +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "hf/arch/init.h" - -#include "hf/api.h" -#include "hf/boot_params.h" -#include "hf/cpio.h" -#include "hf/cpu.h" -#include "hf/dlog.h" -#include "hf/load.h" -#include "hf/mm.h" -#include "hf/mpool.h" -#include "hf/panic.h" -#include "hf/plat/console.h" -#include "hf/std.h" -#include "hf/vm.h" - -#include "vmapi/hf/call.h" - -alignas(alignof( - struct mm_page_table)) char ptable_buf[sizeof(struct mm_page_table) * - HEAP_PAGES]; - -/** - * Performs one-time initialisation of the hypervisor. - */ -static void one_time_init(void) -{ - struct boot_params params; - struct boot_params_update update; - struct memiter primary_initrd; - struct memiter cpio; - void *initrd; - size_t i; - struct mpool ppool; - struct mm_stage1_locked mm_stage1_locked; - - /* Make sure the console is initialised before calling dlog. */ - plat_console_init(); - - dlog("Initialising hafnium\n"); - - arch_one_time_init(); - - mpool_init(&ppool, sizeof(struct mm_page_table)); - mpool_add_chunk(&ppool, ptable_buf, sizeof(ptable_buf)); - - if (!mm_init(&ppool)) { - panic("mm_init failed"); - } - - /* Enable locks now that mm is initialised. */ - dlog_enable_lock(); - mpool_enable_locks(); - - mm_stage1_locked = mm_lock_stage1(); - - if (!plat_get_boot_params(mm_stage1_locked, ¶ms, &ppool)) { - panic("unable to retrieve boot params"); - } - - cpu_module_init(params.cpu_ids, params.cpu_count); - - for (i = 0; i < params.mem_ranges_count; ++i) { - dlog("Memory range: 0x%x - 0x%x\n", - pa_addr(params.mem_ranges[i].begin), - pa_addr(params.mem_ranges[i].end) - 1); - } - - dlog("Ramdisk range: 0x%x - 0x%x\n", pa_addr(params.initrd_begin), - pa_addr(params.initrd_end) - 1); - - /* Map initrd in, and initialise cpio parser. */ - initrd = mm_identity_map(mm_stage1_locked, params.initrd_begin, - params.initrd_end, MM_MODE_R, &ppool); - if (!initrd) { - panic("unable to map initrd in"); - } - - memiter_init(&cpio, initrd, - pa_difference(params.initrd_begin, params.initrd_end)); - - /* Load all VMs. */ - if (!load_primary(mm_stage1_locked, &cpio, params.kernel_arg, - &primary_initrd, &ppool)) { - panic("unable to load primary VM"); - } - - /* - * load_secondary will add regions assigned to the secondary VMs from - * mem_ranges to reserved_ranges. - */ - update.initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next)); - update.initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit)); - update.reserved_ranges_count = 0; - if (!load_secondary(mm_stage1_locked, &cpio, ¶ms, &update, - &ppool)) { - panic("unable to load secondary VMs"); - } - - /* Prepare to run by updating bootparams as seen by primary VM. */ - if (!plat_update_boot_params(mm_stage1_locked, &update, &ppool)) { - panic("plat_update_boot_params failed"); - } - - mm_defrag(mm_stage1_locked, &ppool); - mm_unlock_stage1(&mm_stage1_locked); - - /* Initialise the API page pool. ppool will be empty from now on. */ - api_init(&ppool); - - /* Enable TLB invalidation for VM page table updates. */ - mm_vm_enable_invalidation(); - - dlog("Hafnium initialisation completed\n"); -} - -/** - * The entry point of CPUs when they are turned on. It is supposed to initialise - * all state and return the first vCPU to run. - */ -struct vcpu *cpu_main(struct cpu *c) -{ - struct vcpu *vcpu; - struct vm *vm; - - /* - * Do global one-time initialisation just once. We avoid using atomics - * by only touching the variable from cpu 0. - */ - static volatile bool inited = false; - - if (cpu_index(c) == 0 && !inited) { - inited = true; - one_time_init(); - } - - if (!mm_cpu_init()) { - panic("mm_cpu_init failed"); - } - - vcpu = vm_get_vcpu(vm_find(HF_PRIMARY_VM_ID), cpu_index(c)); - vm = vcpu_get_vm(vcpu); - vcpu_set_cpu(vcpu, c); - - /* Reset the registers to give a clean start for the primary's vCPU. */ - arch_regs_reset(vcpu_get_regs(vcpu), true, vm_get_id(vm), c->id, vm_get_ptable(vm)->root); - - return vcpu; -}