From 00a1f2d05de9b871e7512bd793db94e812789ada Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sat, 31 Aug 2019 03:52:01 +0900 Subject: [PATCH 01/45] Oxidize load.c and main.c --- hfo2/src/arch/fake.rs | 26 +++ hfo2/src/arch/mod.rs | 3 + hfo2/src/boot_params.rs | 59 +++++++ hfo2/src/init.rs | 180 +++++++++++++++++++++ hfo2/src/layout.rs | 38 +++++ hfo2/src/lib.rs | 4 + hfo2/src/load.rs | 342 ++++++++++++++++++++++++++++++++++++++++ hfo2/src/memiter.rs | 4 +- hfo2/src/mm.rs | 8 +- hfo2/src/types.rs | 8 +- src/BUILD.gn | 2 - src/load.c | 323 ------------------------------------- src/main.c | 167 -------------------- 13 files changed, 660 insertions(+), 504 deletions(-) create mode 100644 hfo2/src/boot_params.rs create mode 100644 hfo2/src/init.rs create mode 100644 hfo2/src/layout.rs create mode 100644 hfo2/src/load.rs delete mode 100644 src/load.c delete mode 100644 src/main.c diff --git a/hfo2/src/arch/fake.rs b/hfo2/src/arch/fake.rs index d8d5d9b52..6c30c60a0 100644 --- a/hfo2/src/arch/fake.rs +++ b/hfo2/src/arch/fake.rs @@ -17,6 +17,9 @@ // TODO: Refactor type names and remove this. #![allow(non_camel_case_types)] +use crate::boot_params::*; +use crate::mm::*; +use crate::mpool::*; use crate::types::*; /// The integer type corresponding to the native register size. @@ -51,3 +54,26 @@ pub struct ArchRegs { pub fn arch_cpu_module_init() { // Do nothing. } + +#[no_mangle] +pub extern "C" fn arch_one_time_init() { + unreachable!(); +} + +#[no_mangle] +pub fn plat_get_boot_params( + _stage1_locked: mm_stage1_locked, + _p: *mut BootParams, + _ppool: *mut MPool, +) -> bool { + unreachable!(); +} + +#[no_mangle] +pub fn plat_update_boot_params( + _stage1_locked: mm_stage1_locked, + _p: *mut BootParamsUpdate, + _ppool: *mut MPool, +) -> bool { + unreachable!(); +} diff --git a/hfo2/src/arch/mod.rs b/hfo2/src/arch/mod.rs index 08b4b7ac6..ae2d204be 100644 --- a/hfo2/src/arch/mod.rs +++ b/hfo2/src/arch/mod.rs @@ -14,7 +14,10 @@ * limitations under the License. */ +#[cfg(target_arch = "aarch64")] mod aarch64; + +#[cfg(target_arch = "x86_64")] mod fake; #[cfg(target_arch = "aarch64")] diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs new file mode 100644 index 000000000..73d18514b --- /dev/null +++ b/hfo2/src/boot_params.rs @@ -0,0 +1,59 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::addr::*; +use crate::arch::*; +use crate::mm::*; +use crate::mpool::*; +use crate::types::*; + +pub const MAX_MEM_RANGES: usize = 20; + +#[derive(Clone)] +pub struct MemRange { + pub begin: paddr_t, + pub end: paddr_t, +} + +pub struct BootParams { + pub cpu_ids: [cpu_id_t; MAX_CPUS], + pub cpu_count: usize, + pub mem_ranges: [MemRange; MAX_MEM_RANGES], + pub mem_ranges_count: usize, + pub initrd_begin: paddr_t, + pub initrd_end: paddr_t, + pub kernel_arg: uintreg_t, +} + +pub struct BootParamsUpdate { + pub reserved_ranges: [MemRange; MAX_MEM_RANGES], + pub reserved_ranges_count: usize, + pub initrd_begin: paddr_t, + pub initrd_end: paddr_t, +} + +extern "C" { + pub fn plat_get_boot_params( + stage1_locked: mm_stage1_locked, + p: *mut BootParams, + ppool: *mut MPool, + ) -> bool; + pub fn plat_update_boot_params( + stage1_locked: mm_stage1_locked, + p: *mut BootParamsUpdate, + ppool: *mut MPool, + ) -> bool; +} diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs new file mode 100644 index 000000000..cbf70b722 --- /dev/null +++ b/hfo2/src/init.rs @@ -0,0 +1,180 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem::{self, MaybeUninit}; + +use crate::addr::*; +use crate::api::*; +use crate::boot_params::*; +use crate::cpu::*; +use crate::load::*; +use crate::memiter::*; +use crate::mm::*; +use crate::mpool::*; +use crate::page::*; +use crate::types::*; +use crate::vm::*; + +extern "C" { + fn plat_console_init(); + fn arch_one_time_init(); + fn dlog_enable_lock(); +} + +/// Note(HfO2): this variable was originally of type +/// MaybeUninit<[u8; mem::size_of::() * HEAP_PAGES]>, +/// but it was not aligned to PAGE_SIZE. +static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); + +/// Performs one-time initialisation of the hypervisor. +unsafe fn one_time_init() { + let mut ppool = mem::uninitialized(); + + // Make sure the console is initialised before calling dlog. + plat_console_init(); + + dlog!("Initialising hafnium\n"); + + arch_one_time_init(); + + mpool_init(&mut ppool, PAGE_SIZE); + mpool_add_chunk( + &mut ppool, + PTABLE_BUF.get_mut().as_mut_ptr() as usize as *mut _, + mem::size_of_val(PTABLE_BUF.get_ref()), + ); + + if !mm_init(&mut ppool) { + panic!("mm_init failed"); + } + + // Enable locks now that mm is initialised. + dlog_enable_lock(); + mpool_enable_locks(); + + let mut mm_stage1_locked = mm_lock_stage1(); + let mut params = mem::uninitialized(); + + if !plat_get_boot_params(mm_stage1_locked, &mut params, &mut ppool) { + panic!("unable to retrieve boot params"); + } + + cpu_module_init(&mut params.cpu_ids[0], params.cpu_count); + + for i in 0..params.mem_ranges_count { + dlog!( + "Memory range: 0x{:x} - 0x{:x}\n", + pa_addr(params.mem_ranges[i].begin), + pa_addr(params.mem_ranges[i].end) - 1 + ); + } + + dlog!( + "Ramdisk range: 0x{:x} - 0x{:x}\n", + pa_addr(params.initrd_begin), + pa_addr(params.initrd_end) - 1 + ); + + // Map initrd in, and initialise cpio parser. + let initrd = mm_identity_map( + mm_stage1_locked, + params.initrd_begin, + params.initrd_end, + Mode::R, + &ppool, + ); + if initrd.is_null() { + panic!("unable to map initrd in"); + } + + let mut cpio = mem::uninitialized(); + memiter_init( + &mut cpio, + initrd as *mut _ as usize as *mut _, + pa_difference(params.initrd_begin, params.initrd_end), + ); + + let mut primary_initrd = mem::uninitialized(); + + // Load all VMs. + if !load_primary( + mm_stage1_locked, + &cpio, + params.kernel_arg, + &mut primary_initrd, + &mut ppool, + ) { + panic!("unable to load primary VM"); + } + + let mut update: BootParamsUpdate = mem::uninitialized(); + + // load_secondary will add regions assigned to the secondary VMs from + // mem_ranges to reserved_ranges. + update.initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next as usize as *const _)); + update.initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)); + update.reserved_ranges_count = 0; + if !load_secondary(mm_stage1_locked, &cpio, ¶ms, &mut update, &mut ppool) { + panic!("unable to load secondary VMs"); + } + + // Prepare to run by updating bootparams as seen by primary VM. + if !plat_update_boot_params(mm_stage1_locked, &mut update, &mut ppool) { + panic!("plat_update_boot_params failed"); + } + + mm_defrag(mm_stage1_locked, &ppool); + mm_unlock_stage1(&mut mm_stage1_locked); + + // Initialise the API page pool. ppool will be empty from now on. + api_init(&ppool); + + // Enable TLB invalidation for VM page table updates. + mm_vm_enable_invalidation(); + + dlog!("Hafnium initialisation completed\n"); +} + +static mut INITED: bool = false; + +// The entry point of CPUs when they are turned on. It is supposed to initialise +// all state and return the first vCPU to run. +#[no_mangle] +pub unsafe extern "C" fn cpu_main(c: *mut Cpu) -> *mut VCpu { + // Do global one-time initialisation just once. We avoid using atomics by + // only touching the variable from cpu 0. + if cpu_index(c) == 0 && !INITED { + INITED = true; + one_time_init(); + } + + if !mm_cpu_init() { + panic!("mm_cpu_init failed"); + } + + let vcpu = vm_get_vcpu(vm_find(HF_PRIMARY_VM_ID), cpu_index(c) as spci_vcpu_index_t); + let vm = (*vcpu).vm; + vcpu_set_cpu(vcpu, c); + + // Reset the registers to give a clean start for the primary's vCPU. + (*vcpu) + .inner + .get_mut_unchecked() + .regs + .reset(true, &*vm, (*c).id); + + vcpu +} diff --git a/hfo2/src/layout.rs b/hfo2/src/layout.rs new file mode 100644 index 000000000..723491941 --- /dev/null +++ b/hfo2/src/layout.rs @@ -0,0 +1,38 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::addr::*; + +extern "C" { + pub fn layout_text_begin() -> paddr_t; + pub fn layout_text_end() -> paddr_t; + + pub fn layout_rodata_begin() -> paddr_t; + pub fn layout_rodata_end() -> paddr_t; + + pub fn layout_data_begin() -> paddr_t; + pub fn layout_data_end() -> paddr_t; + + pub fn layout_initrd_begin() -> paddr_t; + pub fn layout_initrd_end() -> paddr_t; + + pub fn layout_fdt_begin() -> paddr_t; + pub fn layout_fdt_end() -> paddr_t; + + pub fn layout_image_end() -> paddr_t; + + pub fn layout_primary_begin() -> paddr_t; +} diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index 709f5d22f..b9116c01d 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -44,7 +44,11 @@ mod abi; mod addr; mod api; mod arch; +mod boot_params; mod cpu; +mod init; +mod layout; +mod load; mod memiter; mod mm; mod mpool; diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs new file mode 100644 index 000000000..05f53db95 --- /dev/null +++ b/hfo2/src/load.rs @@ -0,0 +1,342 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem; +use core::ptr; +use core::slice; + +use crate::addr::*; +use crate::arch::*; +use crate::boot_params::*; +use crate::cpio::*; +use crate::cpu::*; +use crate::layout::*; +use crate::memiter::*; +use crate::mm::*; +use crate::mpool::*; +use crate::page::*; +use crate::std::*; +use crate::types::*; +use crate::utils::*; +use crate::vm::*; + +/// Copies data to an unmapped location by mapping it for write, copying the +/// data, then unmapping it. +/// +/// The data is written so that it is available to all cores with the cache +/// disabled. When switching to the partitions, the caching is initially +/// disabled so the data must be available without the cache. +unsafe fn copy_to_unmapped( + stage1_locked: mm_stage1_locked, + to: paddr_t, + from: *const c_void, + size: usize, + ppool: *mut MPool, +) -> bool { + let to_end = pa_add(to, size); + + let ptr = mm_identity_map(stage1_locked, to, to_end, Mode::W, ppool); + if ptr.is_null() { + return false; + } + + memcpy_s(ptr as usize as *mut _, size, from, size); + arch_mm_write_back_dcache(ptr as usize, size); + + mm_unmap(stage1_locked, to, to_end, ppool); + + true +} + +/// Loads the primary VM. +pub unsafe fn load_primary( + stage1_locked: mm_stage1_locked, + cpio: *const MemIter, + kernel_arg: uintreg_t, + initrd: *mut MemIter, + ppool: *mut MPool, +) -> bool { + let mut it = mem::uninitialized(); + let primary_begin = layout_primary_begin(); + + if !cpio_find_file(cpio, "vmlinuz\0".as_ptr(), &mut it) { + dlog!("Unable to find vmlinuz\n"); + return false; + } + + dlog!( + "Copying primary to {:p}\n", + pa_addr(primary_begin) as *const u8 + ); + if !copy_to_unmapped( + stage1_locked, + primary_begin, + it.next as usize as *mut _, + it.limit.offset_from(it.next) as usize, + ppool, + ) { + dlog!("Unable to relocate kernel for primary vm.\n"); + return false; + } + + if !cpio_find_file(cpio, "initrd.img\0".as_ptr(), initrd) { + dlog!("Unable to find initrd.img\n"); + return false; + } + + { + let mut vm = mem::uninitialized(); + if !vm_init(MAX_CPUS as u16, ppool, &mut vm) { + dlog!("Unable to initialise primary vm\n"); + return false; + } + + if (*vm).id != HF_PRIMARY_VM_ID { + dlog!("Primary vm was not given correct id\n"); + return false; + } + + // Map the 1TB of memory. + // TODO: We should do a whitelist rather than blacklist. + if !mm_vm_identity_map( + &mut (*vm).inner.get_mut_unchecked().ptable, + pa_init(0), + pa_init(1024usize * 1024 * 1024 * 1024), + Mode::R | Mode::W | Mode::X, + ptr::null_mut(), + ppool, + ) { + dlog!("Unable to initialise memory for primary vm\n"); + return false; + } + + if !mm_vm_unmap_hypervisor(&mut (*vm).inner.get_mut_unchecked().ptable, ppool) { + dlog!("Unable to unmap hypervisor from primary vm\n"); + return false; + } + + (*vm_get_vcpu(vm, 0)) + .inner + .lock() + .on(ipa_from_pa(primary_begin), kernel_arg); + } + + true +} + +/// Try to find a memory range of the given size within the given ranges, and +/// remove it from them. Return true on success, or false if no large enough +/// contiguous range is found. +unsafe fn carve_out_mem_range( + mem_ranges: *mut MemRange, + mem_ranges_count: usize, + size_to_find: u64, + found_begin: *mut paddr_t, + found_end: *mut paddr_t, +) -> bool { + // TODO(b/116191358): Consider being cleverer about how we pack VMs + // together, with a non-greedy algorithm. + for i in 0..mem_ranges_count as isize { + if size_to_find + <= pa_difference((*mem_ranges.offset(i)).begin, (*mem_ranges.offset(i)).end) as u64 + { + // This range is big enough, take some of it from the end and reduce + // its size accordingly. + *found_end = (*mem_ranges.offset(i)).end; + *found_begin = pa_init(pa_addr((*mem_ranges.offset(i)).end) - size_to_find as usize); + (*mem_ranges.offset(i)).end = *found_begin; + return true; + } + } + + false +} + +/// Given arrays of memory ranges before and after memory was removed for +/// secondary VMs, add the difference to the reserved ranges of the given +/// update. Return true on success, or false if there would be more than +/// MAX_MEM_RANGES reserved ranges after adding the new ones. +/// `before` and `after` must be arrays of exactly `mem_ranges_count` elements. +unsafe fn update_reserved_ranges( + update: *mut BootParamsUpdate, + before: *const MemRange, + after: *const MemRange, + mem_ranges_count: usize, +) -> bool { + let after = slice::from_raw_parts(after, mem_ranges_count); + let before = slice::from_raw_parts(before, mem_ranges_count); + + for (before, after) in before.iter().zip(after.iter()) { + if pa_addr(after.begin) > pa_addr(before.begin) { + if (*update).reserved_ranges_count >= MAX_MEM_RANGES { + dlog!("Too many reserved ranges after loading secondary VMs.\n"); + return false; + } + + (*update).reserved_ranges[(*update).reserved_ranges_count].begin = after.end; + (*update).reserved_ranges[(*update).reserved_ranges_count].end = before.end; + (*update).reserved_ranges_count += 1; + } + } + + true +} + +/// Loads all secondary VMs into the memory ranges from the given params. +/// Memory reserved for the VMs is added to the `reserved_ranges` of `update`. +pub unsafe fn load_secondary( + stage1_locked: mm_stage1_locked, + cpio: *const MemIter, + params: *const BootParams, + update: *mut BootParamsUpdate, + ppool: *mut MPool, +) -> bool { + let mut mem_ranges_available: [MemRange; MAX_MEM_RANGES] = mem::uninitialized(); + // static_assert( + // sizeof(mem_ranges_available) == sizeof(params->mem_ranges), + // "mem_range arrays must be the same size for memcpy."); + + const_assert!(mem::size_of::() * MAX_MEM_RANGES < 500); + mem_ranges_available.clone_from_slice(&(*params).mem_ranges); + + let primary = vm_find(HF_PRIMARY_VM_ID); + let mut it = mem::uninitialized(); + + if !cpio_find_file(cpio, "vms.txt\0".as_ptr(), &mut it) { + dlog!("vms.txt is missing\n"); + return true; + } + + // Round the last addresses down to the page size. + for i in 0..(*params).mem_ranges_count { + mem_ranges_available[i].end = + pa_init(round_down(pa_addr(mem_ranges_available[i].end), PAGE_SIZE)); + } + + let mut mem = mem::uninitialized(); + let mut cpu = mem::uninitialized(); + let mut name = mem::uninitialized(); + + while memiter_parse_uint(&mut it, &mut mem) + && memiter_parse_uint(&mut it, &mut cpu) + && memiter_parse_str(&mut it, &mut name) + { + dlog!("Loading "); + let mut p = name.next; + while p != name.limit { + dlog!("{}", *p as char); + p = p.offset(1); + } + dlog!("\n"); + + let mut kernel = mem::uninitialized(); + + if !cpio_find_file_memiter(cpio, &name, &mut kernel) { + dlog!("Unable to load kernel\n"); + continue; + } + + // Round up to page size. + mem = (mem + PAGE_SIZE as u64 - 1) & !(PAGE_SIZE as u64 - 1); + + if mem < kernel.limit.offset_from(kernel.next) as u64 { + dlog!("Kernel is larger than available memory\n"); + continue; + } + + let mut secondary_mem_begin = mem::uninitialized(); + let mut secondary_mem_end = mem::uninitialized(); + + if !carve_out_mem_range( + mem_ranges_available.as_mut_ptr(), + (*params).mem_ranges_count, + mem, + &mut secondary_mem_begin, + &mut secondary_mem_end, + ) { + dlog!("Not enough memory ({} bytes)\n", mem); + continue; + } + + if !copy_to_unmapped( + stage1_locked, + secondary_mem_begin, + kernel.next as usize as *const _, + kernel.limit.offset_from(kernel.next) as usize, + ppool, + ) { + dlog!("Unable to copy kernel\n"); + continue; + } + + let mut vm = mem::uninitialized(); + if !vm_init(cpu as spci_vcpu_count_t, ppool, &mut vm) { + dlog!("Unable to initialise VM\n"); + continue; + } + + let mut secondary_entry = mem::uninitialized(); + + // Grant the VM access to the memory. + if !mm_vm_identity_map( + &mut (*vm).inner.get_mut_unchecked().ptable, + secondary_mem_begin, + secondary_mem_end, + Mode::R | Mode::W | Mode::X, + &mut secondary_entry, + ppool, + ) { + dlog!("Unable to initialise memory\n"); + continue; + } + + // Deny the primary VM access to this memory. + if !mm_vm_unmap( + &mut (*primary).inner.get_mut_unchecked().ptable, + secondary_mem_begin, + secondary_mem_end, + ppool, + ) { + dlog!("Unable to unmap secondary VM from primary VM\n"); + return false; + } + + dlog!( + "Loaded with {} vcpus, entry at 0x{:x}\n", + cpu, + pa_addr(secondary_mem_begin) + ); + + let vcpu = vm_get_vcpu(vm, 0); + vcpu_secondary_reset_and_start( + vcpu, + secondary_entry, + pa_difference(secondary_mem_begin, secondary_mem_end) as uintreg_t, + ); + } + + // Add newly reserved areas to update params by looking at the difference + // between the available ranges from the original params and the updated + // mem_ranges_available. We assume that the number and order of available + // ranges is the same, i.e. we don't remove any ranges above only make them + // smaller. + update_reserved_ranges( + update, + (*params).mem_ranges.as_ptr(), + mem_ranges_available.as_ptr(), + (*params).mem_ranges_count, + ) +} diff --git a/hfo2/src/memiter.rs b/hfo2/src/memiter.rs index ed1a409ff..980a5de78 100644 --- a/hfo2/src/memiter.rs +++ b/hfo2/src/memiter.rs @@ -22,8 +22,8 @@ use crate::types::*; #[repr(C)] #[derive(Clone)] pub struct MemIter { - next: *const u8, - limit: *const u8, + pub next: *const u8, + pub limit: *const u8, } /// Determines if a character is a whitespace. diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index 01243eae3..cc0a3db19 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -36,6 +36,7 @@ use reduce::Reduce; use crate::addr::*; use crate::arch::*; +use crate::layout::*; use crate::mpool::MPool; use crate::page::*; use crate::spinlock::{SpinLock, SpinLockGuard}; @@ -79,13 +80,6 @@ extern "C" { fn arch_mm_combine_table_entry_attrs(table_attrs: u64, block_attrs: u64) -> u64; fn plat_console_mm_init(stage1_locked: mm_stage1_locked, mpool: *const MPool); - - fn layout_text_begin() -> paddr_t; - fn layout_text_end() -> paddr_t; - fn layout_rodata_begin() -> paddr_t; - fn layout_rodata_end() -> paddr_t; - fn layout_data_begin() -> paddr_t; - fn layout_data_end() -> paddr_t; } bitflags! { diff --git a/hfo2/src/types.rs b/hfo2/src/types.rs index f7ad1d7d3..41ae4a2b2 100644 --- a/hfo2/src/types.rs +++ b/hfo2/src/types.rs @@ -58,9 +58,11 @@ pub const HF_INVALID_INTID: intid_t = 0xffff_ffff; pub const HF_VIRTUAL_TIMER_INTID: intid_t = 3; // These constants are originally from build scripts. Fortunately most -// testing environments have same conditions (MAX_CPUS=8, MAX_VMS=16.) And -// only one environment (host_fake) doesn't but it's for the unit test, so -// works fine under this settting (See //project/reference/BUILD.gn.) +// testing environments have same conditions (HEAP_PAGES=60, MAX_CPUS=8, +// MAX_VMS=16.) And only one environment (host_fake) doesn't but it's for the +// unit test, so works fine under this settting (See +// //project/reference/BUILD.gn.) +pub const HEAP_PAGES: usize = 60; pub const MAX_CPUS: usize = 8; pub const MAX_VMS: usize = 16; diff --git a/src/BUILD.gn b/src/BUILD.gn index 11369d43b..22ce009bf 100644 --- a/src/BUILD.gn +++ b/src/BUILD.gn @@ -29,8 +29,6 @@ hypervisor("hafnium") { # src targets will merge! source_set("src_not_testable_yet") { sources = [ - "load.c", - "main.c", "plat.c", ] libs = ["//hfo2/target/aarch64-hfo2/release/libhfo2.a"] diff --git a/src/load.c b/src/load.c deleted file mode 100644 index 57167d4a8..000000000 --- a/src/load.c +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hf/load.h" - -#include - -#include "hf/api.h" -#include "hf/boot_params.h" -#include "hf/dlog.h" -#include "hf/layout.h" -#include "hf/memiter.h" -#include "hf/mm.h" -#include "hf/plat/console.h" -#include "hf/static_assert.h" -#include "hf/std.h" -#include "hf/vm.h" - -#include "vmapi/hf/call.h" - -/** - * Copies data to an unmapped location by mapping it for write, copying the - * data, then unmapping it. - * - * The data is written so that it is available to all cores with the cache - * disabled. When switching to the partitions, the caching is initially disabled - * so the data must be available without the cache. - */ -static bool copy_to_unmapped(struct mm_stage1_locked stage1_locked, paddr_t to, - const void *from, size_t size, struct mpool *ppool) -{ - paddr_t to_end = pa_add(to, size); - void *ptr; - - ptr = mm_identity_map(stage1_locked, to, to_end, MM_MODE_W, ppool); - if (!ptr) { - return false; - } - - memcpy_s(ptr, size, from, size); - arch_mm_write_back_dcache(ptr, size); - - mm_unmap(stage1_locked, to, to_end, ppool); - - return true; -} - -/** - * Loads the primary VM. - */ -bool load_primary(struct mm_stage1_locked stage1_locked, - const struct memiter *cpio, uintreg_t kernel_arg, - struct memiter *initrd, struct mpool *ppool) -{ - struct memiter it; - paddr_t primary_begin = layout_primary_begin(); - - if (!cpio_find_file(cpio, "vmlinuz", &it)) { - dlog("Unable to find vmlinuz\n"); - return false; - } - - dlog("Copying primary to %p\n", pa_addr(primary_begin)); - if (!copy_to_unmapped(stage1_locked, primary_begin, it.next, - it.limit - it.next, ppool)) { - dlog("Unable to relocate kernel for primary vm.\n"); - return false; - } - - if (!cpio_find_file(cpio, "initrd.img", initrd)) { - dlog("Unable to find initrd.img\n"); - return false; - } - - { - struct vm *vm; - struct vcpu_execution_locked vcpu_execution_locked; - - if (!vm_init(MAX_CPUS, ppool, &vm)) { - dlog("Unable to initialise primary vm\n"); - return false; - } - - if (vm_get_id(vm) != HF_PRIMARY_VM_ID) { - dlog("Primary vm was not given correct id\n"); - return false; - } - - /* Map the 1TB of memory. */ - /* TODO: We should do a whitelist rather than a blacklist. */ - if (!mm_vm_identity_map( - vm_get_ptable(vm), pa_init(0), - pa_init(UINT64_C(1024) * 1024 * 1024 * 1024), - MM_MODE_R | MM_MODE_W | MM_MODE_X, NULL, ppool)) { - dlog("Unable to initialise memory for primary vm\n"); - return false; - } - - if (!mm_vm_unmap_hypervisor(vm_get_ptable(vm), ppool)) { - dlog("Unable to unmap hypervisor from primary vm\n"); - return false; - } - - vcpu_execution_locked = vcpu_lock(vm_get_vcpu(vm, 0)); - vcpu_on(vcpu_execution_locked, ipa_from_pa(primary_begin), kernel_arg); - vcpu_unlock(&vcpu_execution_locked); - } - - return true; -} - -/** - * Try to find a memory range of the given size within the given ranges, and - * remove it from them. Return true on success, or false if no large enough - * contiguous range is found. - */ -static bool carve_out_mem_range(struct mem_range *mem_ranges, - size_t mem_ranges_count, uint64_t size_to_find, - paddr_t *found_begin, paddr_t *found_end) -{ - size_t i; - - /* - * TODO(b/116191358): Consider being cleverer about how we pack VMs - * together, with a non-greedy algorithm. - */ - for (i = 0; i < mem_ranges_count; ++i) { - if (size_to_find <= - pa_difference(mem_ranges[i].begin, mem_ranges[i].end)) { - /* - * This range is big enough, take some of it from the - * end and reduce its size accordingly. - */ - *found_end = mem_ranges[i].end; - *found_begin = pa_init(pa_addr(mem_ranges[i].end) - - size_to_find); - mem_ranges[i].end = *found_begin; - return true; - } - } - return false; -} - -/** - * Given arrays of memory ranges before and after memory was removed for - * secondary VMs, add the difference to the reserved ranges of the given update. - * Return true on success, or false if there would be more than MAX_MEM_RANGES - * reserved ranges after adding the new ones. - * `before` and `after` must be arrays of exactly `mem_ranges_count` elements. - */ -static bool update_reserved_ranges(struct boot_params_update *update, - const struct mem_range *before, - const struct mem_range *after, - size_t mem_ranges_count) -{ - size_t i; - - for (i = 0; i < mem_ranges_count; ++i) { - if (pa_addr(after[i].begin) > pa_addr(before[i].begin)) { - if (update->reserved_ranges_count >= MAX_MEM_RANGES) { - dlog("Too many reserved ranges after loading " - "secondary VMs.\n"); - return false; - } - update->reserved_ranges[update->reserved_ranges_count] - .begin = before[i].begin; - update->reserved_ranges[update->reserved_ranges_count] - .end = after[i].begin; - update->reserved_ranges_count++; - } - if (pa_addr(after[i].end) < pa_addr(before[i].end)) { - if (update->reserved_ranges_count >= MAX_MEM_RANGES) { - dlog("Too many reserved ranges after loading " - "secondary VMs.\n"); - return false; - } - update->reserved_ranges[update->reserved_ranges_count] - .begin = after[i].end; - update->reserved_ranges[update->reserved_ranges_count] - .end = before[i].end; - update->reserved_ranges_count++; - } - } - - return true; -} - -/** - * Loads all secondary VMs into the memory ranges from the given params. - * Memory reserved for the VMs is added to the `reserved_ranges` of `update`. - */ -bool load_secondary(struct mm_stage1_locked stage1_locked, - const struct memiter *cpio, - const struct boot_params *params, - struct boot_params_update *update, struct mpool *ppool) -{ - struct vm *primary; - struct memiter it; - struct memiter name; - uint64_t mem; - uint64_t cpu; - struct mem_range mem_ranges_available[MAX_MEM_RANGES]; - size_t i; - - static_assert( - sizeof(mem_ranges_available) == sizeof(params->mem_ranges), - "mem_range arrays must be the same size for memcpy."); - static_assert(sizeof(mem_ranges_available) < 500, - "This will use too much stack, either make " - "MAX_MEM_RANGES smaller or change this."); - memcpy_s(mem_ranges_available, sizeof(mem_ranges_available), - params->mem_ranges, sizeof(params->mem_ranges)); - - primary = vm_find(HF_PRIMARY_VM_ID); - - if (!cpio_find_file(cpio, "vms.txt", &it)) { - dlog("vms.txt is missing\n"); - return true; - } - - /* Round the last addresses down to the page size. */ - for (i = 0; i < params->mem_ranges_count; ++i) { - mem_ranges_available[i].end = pa_init(align_down( - pa_addr(mem_ranges_available[i].end), PAGE_SIZE)); - } - - while (memiter_parse_uint(&it, &mem) && memiter_parse_uint(&it, &cpu) && - memiter_parse_str(&it, &name)) { - struct memiter kernel; - paddr_t secondary_mem_begin; - paddr_t secondary_mem_end; - ipaddr_t secondary_entry; - const char *p; - struct vm *vm; - struct vcpu *vcpu; - - dlog("Loading "); - for (p = name.next; p != name.limit; ++p) { - dlog("%c", *p); - } - dlog("\n"); - - if (!cpio_find_file_memiter(cpio, &name, &kernel)) { - dlog("Unable to load kernel\n"); - continue; - } - - /* Round up to page size. */ - mem = (mem + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - - if (mem < kernel.limit - kernel.next) { - dlog("Kernel is larger than available memory\n"); - continue; - } - - if (!carve_out_mem_range( - mem_ranges_available, params->mem_ranges_count, mem, - &secondary_mem_begin, &secondary_mem_end)) { - dlog("Not enough memory (%u bytes)\n", mem); - continue; - } - - if (!copy_to_unmapped(stage1_locked, secondary_mem_begin, - kernel.next, kernel.limit - kernel.next, - ppool)) { - dlog("Unable to copy kernel\n"); - continue; - } - - if (!vm_init(cpu, ppool, &vm)) { - dlog("Unable to initialise VM\n"); - continue; - } - - /* Grant the VM access to the memory. */ - if (!mm_vm_identity_map(vm_get_ptable(vm), secondary_mem_begin, - secondary_mem_end, - MM_MODE_R | MM_MODE_W | MM_MODE_X, - &secondary_entry, ppool)) { - dlog("Unable to initialise memory\n"); - continue; - } - - /* Deny the primary VM access to this memory. */ - if (!mm_vm_unmap(vm_get_ptable(primary), secondary_mem_begin, - secondary_mem_end, ppool)) { - dlog("Unable to unmap secondary VM from primary VM\n"); - return false; - } - - dlog("Loaded with %u vcpus, entry at 0x%x\n", cpu, - pa_addr(secondary_mem_begin)); - - vcpu = vm_get_vcpu(vm, 0); - vcpu_secondary_reset_and_start( - vcpu, secondary_entry, - pa_difference(secondary_mem_begin, secondary_mem_end)); - } - - /* - * Add newly reserved areas to update params by looking at the - * difference between the available ranges from the original params and - * the updated mem_ranges_available. We assume that the number and order - * of available ranges is the same, i.e. we don't remove any ranges - * above only make them smaller. - */ - return update_reserved_ranges(update, params->mem_ranges, - mem_ranges_available, - params->mem_ranges_count); -} diff --git a/src/main.c b/src/main.c deleted file mode 100644 index 6695292cc..000000000 --- a/src/main.c +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "hf/arch/init.h" - -#include "hf/api.h" -#include "hf/boot_params.h" -#include "hf/cpio.h" -#include "hf/cpu.h" -#include "hf/dlog.h" -#include "hf/load.h" -#include "hf/mm.h" -#include "hf/mpool.h" -#include "hf/panic.h" -#include "hf/plat/console.h" -#include "hf/std.h" -#include "hf/vm.h" - -#include "vmapi/hf/call.h" - -alignas(alignof( - struct mm_page_table)) char ptable_buf[sizeof(struct mm_page_table) * - HEAP_PAGES]; - -/** - * Performs one-time initialisation of the hypervisor. - */ -static void one_time_init(void) -{ - struct boot_params params; - struct boot_params_update update; - struct memiter primary_initrd; - struct memiter cpio; - void *initrd; - size_t i; - struct mpool ppool; - struct mm_stage1_locked mm_stage1_locked; - - /* Make sure the console is initialised before calling dlog. */ - plat_console_init(); - - dlog("Initialising hafnium\n"); - - arch_one_time_init(); - - mpool_init(&ppool, sizeof(struct mm_page_table)); - mpool_add_chunk(&ppool, ptable_buf, sizeof(ptable_buf)); - - if (!mm_init(&ppool)) { - panic("mm_init failed"); - } - - /* Enable locks now that mm is initialised. */ - dlog_enable_lock(); - mpool_enable_locks(); - - mm_stage1_locked = mm_lock_stage1(); - - if (!plat_get_boot_params(mm_stage1_locked, ¶ms, &ppool)) { - panic("unable to retrieve boot params"); - } - - cpu_module_init(params.cpu_ids, params.cpu_count); - - for (i = 0; i < params.mem_ranges_count; ++i) { - dlog("Memory range: 0x%x - 0x%x\n", - pa_addr(params.mem_ranges[i].begin), - pa_addr(params.mem_ranges[i].end) - 1); - } - - dlog("Ramdisk range: 0x%x - 0x%x\n", pa_addr(params.initrd_begin), - pa_addr(params.initrd_end) - 1); - - /* Map initrd in, and initialise cpio parser. */ - initrd = mm_identity_map(mm_stage1_locked, params.initrd_begin, - params.initrd_end, MM_MODE_R, &ppool); - if (!initrd) { - panic("unable to map initrd in"); - } - - memiter_init(&cpio, initrd, - pa_difference(params.initrd_begin, params.initrd_end)); - - /* Load all VMs. */ - if (!load_primary(mm_stage1_locked, &cpio, params.kernel_arg, - &primary_initrd, &ppool)) { - panic("unable to load primary VM"); - } - - /* - * load_secondary will add regions assigned to the secondary VMs from - * mem_ranges to reserved_ranges. - */ - update.initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next)); - update.initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit)); - update.reserved_ranges_count = 0; - if (!load_secondary(mm_stage1_locked, &cpio, ¶ms, &update, - &ppool)) { - panic("unable to load secondary VMs"); - } - - /* Prepare to run by updating bootparams as seen by primary VM. */ - if (!plat_update_boot_params(mm_stage1_locked, &update, &ppool)) { - panic("plat_update_boot_params failed"); - } - - mm_defrag(mm_stage1_locked, &ppool); - mm_unlock_stage1(&mm_stage1_locked); - - /* Initialise the API page pool. ppool will be empty from now on. */ - api_init(&ppool); - - /* Enable TLB invalidation for VM page table updates. */ - mm_vm_enable_invalidation(); - - dlog("Hafnium initialisation completed\n"); -} - -/** - * The entry point of CPUs when they are turned on. It is supposed to initialise - * all state and return the first vCPU to run. - */ -struct vcpu *cpu_main(struct cpu *c) -{ - struct vcpu *vcpu; - struct vm *vm; - - /* - * Do global one-time initialisation just once. We avoid using atomics - * by only touching the variable from cpu 0. - */ - static volatile bool inited = false; - - if (cpu_index(c) == 0 && !inited) { - inited = true; - one_time_init(); - } - - if (!mm_cpu_init()) { - panic("mm_cpu_init failed"); - } - - vcpu = vm_get_vcpu(vm_find(HF_PRIMARY_VM_ID), cpu_index(c)); - vm = vcpu_get_vm(vcpu); - vcpu_set_cpu(vcpu, c); - - /* Reset the registers to give a clean start for the primary's vCPU. */ - arch_regs_reset(vcpu_get_regs(vcpu), true, vm_get_id(vm), c->id, vm_get_ptable(vm)->root); - - return vcpu; -} From bc7ca9f3a2c5325182f0e332347d74724d58535c Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sat, 31 Aug 2019 21:02:50 +0900 Subject: [PATCH 02/45] Oxidize unit test code (abi_test.cc) --- hfo2/src/abi.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ hfo2/src/init.rs | 3 +- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/hfo2/src/abi.rs b/hfo2/src/abi.rs index c27569f83..e592eaf24 100644 --- a/hfo2/src/abi.rs +++ b/hfo2/src/abi.rs @@ -115,3 +115,84 @@ impl TryFrom for HfShare { } } } + +#[cfg(test)] +mod test { + use super::*; + + /// Encode a preempted response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_preemptd() { + let res = HfVCpuRunReturn::Preempted; + assert_eq!(res.into_raw(), 0); + } + + /// Encode a yield response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_yield() { + let res = HfVCpuRunReturn::Yield; + assert_eq!(res.into_raw(), 1); + } + + /// Encode wait-for-interrupt response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_interrupt() { + let res = HfVCpuRunReturn::WaitForInterrupt { + ns: HF_SLEEP_INDEFINITE + }; + assert_eq!(res.into_raw(), 0xffffffffffffff02); + } + + /// Encoding wait-for-interrupt response with too large sleep duration will + /// drop the top octet. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_interrupt_sleep_too_long() { + let res = HfVCpuRunReturn::WaitForInterrupt { + ns: 0xcc22888888888888, + }; + assert_eq!(res.into_raw(), 0x2288888888888802); + } + + /// Encode wait-for-message response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_message() { + let res = HfVCpuRunReturn::WaitForMessage { + ns: HF_SLEEP_INDEFINITE, + }; + assert_eq!(res.into_raw(), 0xffffffffffffff03); + } + + /// Encoding wait-for-message response with too large sleep duration will + /// drop the top octet. + #[test] + fn abi_hf_vcpu_run_return_encode_wait_for_message_sleep_too_long() { + let res = HfVCpuRunReturn::WaitForMessage { + ns: 0xaa99777777777777, + }; + assert_eq!(res.into_raw(), 0x9977777777777703); + } + + /// Encode wake up response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_wake_up() { + let res = HfVCpuRunReturn::WakeUp { + vm_id: 0x1234, + vcpu: 0xabcd, + }; + assert_eq!(res.into_raw(), 0x1234abcd0004); + } + + /// Encode a 'notify waiters' response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_notify_waiters() { + let res = HfVCpuRunReturn::NotifyWaiters; + assert_eq!(res.into_raw(), 6); + } + + /// Encode an aborted response without leaking. + #[test] + fn abi_hf_vcpu_run_return_encode_aborted() { + let res = HfVCpuRunReturn::Aborted; + assert_eq!(res.into_raw(), 7); + } +} diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index cbf70b722..d099c28ca 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -41,7 +41,6 @@ static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit( /// Performs one-time initialisation of the hypervisor. unsafe fn one_time_init() { - let mut ppool = mem::uninitialized(); // Make sure the console is initialised before calling dlog. plat_console_init(); @@ -50,7 +49,7 @@ unsafe fn one_time_init() { arch_one_time_init(); - mpool_init(&mut ppool, PAGE_SIZE); + let mut ppool = MPool::new(); mpool_add_chunk( &mut ppool, PTABLE_BUF.get_mut().as_mut_ptr() as usize as *mut _, From 2636ccca6a971e1e785828566e3b3488aafed771 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sat, 31 Aug 2019 21:23:05 +0900 Subject: [PATCH 03/45] Remove c++ test codes and add `cargo test` in test script. --- inc/vmapi/hf/abi.h | 26 --------- kokoro/ubuntu/test.sh | 2 + src/abi_test.cc | 121 ------------------------------------------ 3 files changed, 2 insertions(+), 147 deletions(-) diff --git a/inc/vmapi/hf/abi.h b/inc/vmapi/hf/abi.h index 538a42e5b..4682f9c21 100644 --- a/inc/vmapi/hf/abi.h +++ b/inc/vmapi/hf/abi.h @@ -121,32 +121,6 @@ enum hf_share { HF_MEMORY_SHARE, }; -/** - * Encode an hf_vcpu_run_return struct in the 64-bit packing ABI. - */ -static inline uint64_t hf_vcpu_run_return_encode(struct hf_vcpu_run_return res) -{ - uint64_t ret = res.code & 0xff; - - switch (res.code) { - case HF_VCPU_RUN_WAKE_UP: - ret |= (uint64_t)res.wake_up.vm_id << 32; - ret |= (uint64_t)res.wake_up.vcpu << 16; - break; - case HF_VCPU_RUN_MESSAGE: - ret |= res.message.vm_id << 8; - break; - case HF_VCPU_RUN_WAIT_FOR_INTERRUPT: - case HF_VCPU_RUN_WAIT_FOR_MESSAGE: - ret |= res.sleep.ns << 8; - break; - default: - break; - } - - return ret; -} - /** * Decode an hf_vcpu_run_return struct from the 64-bit packing ABI. */ diff --git a/kokoro/ubuntu/test.sh b/kokoro/ubuntu/test.sh index c29a1fe45..9c22fdb46 100755 --- a/kokoro/ubuntu/test.sh +++ b/kokoro/ubuntu/test.sh @@ -61,6 +61,8 @@ $TIMEOUT 30s $OUT/host_fake_clang/unit_tests \ --gtest_output="xml:$OUT/kokoro_log/unit_tests/sponge_log.xml" \ | tee $OUT/kokoro_log/unit_tests/sponge_log.log +cd hfo2 && cargo test && cd .. + $HFTEST arch_test $HFTEST hafnium --initrd test/vmapi/gicv3/gicv3_test $HFTEST hafnium --initrd test/vmapi/primary_only/primary_only_test diff --git a/src/abi_test.cc b/src/abi_test.cc index 204472d4f..d5b3968bf 100644 --- a/src/abi_test.cc +++ b/src/abi_test.cc @@ -24,28 +24,6 @@ namespace { using ::testing::Eq; -/** - * Simulate an uninitialized hf_vcpu_run_return so it can be detected if any - * uninitialized fields make their way into the encoded form which would - * indicate a data leak. - */ -struct hf_vcpu_run_return dirty_vcpu_run_return() -{ - struct hf_vcpu_run_return res; - memset(&res, 0xc5, sizeof(res)); - return res; -} - -/** - * Encode a preempted response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_preempted) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_PREEMPTED; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0)); -} - /** * Decode a preempted response ignoring the irrelevant bits. */ @@ -56,16 +34,6 @@ TEST(abi, hf_vcpu_run_return_decode_preempted) EXPECT_THAT(res.code, Eq(HF_VCPU_RUN_PREEMPTED)); } -/** - * Encode a yield response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_yield) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_YIELD; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(1)); -} - /** * Decode a yield response ignoring the irrelevant bits. */ @@ -76,29 +44,6 @@ TEST(abi, hf_vcpu_run_return_decode_yield) EXPECT_THAT(res.code, Eq(HF_VCPU_RUN_YIELD)); } -/** - * Encode wait-for-interrupt response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_interrupt) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT; - res.sleep.ns = HF_SLEEP_INDEFINITE; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0xffffffffffffff02)); -} - -/** - * Encoding wait-for-interrupt response with too large sleep duration will drop - * the top octet. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_interrupt_sleep_too_long) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_INTERRUPT; - res.sleep.ns = 0xcc22888888888888; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x2288888888888802)); -} - /** * Decode a wait-for-interrupt response ignoring the irrelevant bits. */ @@ -110,29 +55,6 @@ TEST(abi, hf_vcpu_run_return_decode_wait_for_interrupt) EXPECT_THAT(res.sleep.ns, Eq(0x1234abcdbadb01)); } -/** - * Encode wait-for-message response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_message) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_MESSAGE; - res.sleep.ns = HF_SLEEP_INDEFINITE; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0xffffffffffffff03)); -} - -/** - * Encoding wait-for-message response with too large sleep duration will drop - * the top octet. - */ -TEST(abi, hf_vcpu_run_return_encode_wait_for_message_sleep_too_long) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAIT_FOR_MESSAGE; - res.sleep.ns = 0xaa99777777777777; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x9977777777777703)); -} - /** * Decode a wait-for-message response ignoring the irrelevant bits. */ @@ -144,18 +66,6 @@ TEST(abi, hf_vcpu_run_return_decode_wait_for_message) EXPECT_THAT(res.sleep.ns, Eq(0x12347654badb01)); } -/** - * Encode wake up response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_wake_up) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_WAKE_UP; - res.wake_up.vm_id = 0x1234; - res.wake_up.vcpu = 0xabcd; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x1234abcd0004)); -} - /** * Decode a wake up response ignoring the irrelevant bits. */ @@ -168,17 +78,6 @@ TEST(abi, hf_vcpu_run_return_decode_wake_up) EXPECT_THAT(res.wake_up.vcpu, Eq(0xf00d)); } -/** - * Encode message response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_message) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_MESSAGE; - res.message.vm_id = 0xf007; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(0x0000000000f00705)); -} - /** * Decode a wake up response ignoring the irrelevant bits. */ @@ -190,16 +89,6 @@ TEST(abi, hf_vcpu_run_return_decode_message) EXPECT_THAT(res.message.vm_id, Eq(0x9162)); } -/** - * Encode a 'notify waiters' response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_notify_waiters) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_NOTIFY_WAITERS; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(6)); -} - /** * Decode a 'notify waiters' response ignoring the irrelevant bits. */ @@ -210,16 +99,6 @@ TEST(abi, hf_vcpu_run_return_decode_notify_waiters) EXPECT_THAT(res.code, Eq(HF_VCPU_RUN_NOTIFY_WAITERS)); } -/** - * Encode an aborted response without leaking. - */ -TEST(abi, hf_vcpu_run_return_encode_aborted) -{ - struct hf_vcpu_run_return res = dirty_vcpu_run_return(); - res.code = HF_VCPU_RUN_ABORTED; - EXPECT_THAT(hf_vcpu_run_return_encode(res), Eq(7)); -} - /** * Decode an aborted response ignoring the irrelevant bits. */ From 30e6ded0b252092f85ed9c2746ea7d21820b5c05 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sat, 31 Aug 2019 17:27:13 +0900 Subject: [PATCH 04/45] Oxidize FDT --- hfo2/src/fdt.rs | 374 ++++++++++++++++++++++++++++++++++++++++ hfo2/src/fdt_handler.rs | 108 ++++++++++++ hfo2/src/lib.rs | 2 + hfo2/src/plat.rs | 113 ++++++++++++ 4 files changed, 597 insertions(+) create mode 100644 hfo2/src/fdt.rs create mode 100644 hfo2/src/fdt_handler.rs create mode 100644 hfo2/src/plat.rs diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs new file mode 100644 index 000000000..7f7018f0c --- /dev/null +++ b/hfo2/src/fdt.rs @@ -0,0 +1,374 @@ +/* + * Copyright 2019 Sanguk Park. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem; +use core::ptr; +use core::slice; +use core::str; + +use crate::utils::*; +use crate::std::*; + +use scopeguard::guard; + +extern "C" { + fn strcmp(a: *const u8, b: *const u8) -> i64; +} + +#[derive(Clone)] +pub struct FdtNode { + hdr: *const FdtHeader, + begin: *const u8, + end: *const u8, + strs: *const u8, +} + +#[repr(C)] +struct FdtHeader { + magic: u32, + totalsize: u32, + off_dt_struct: u32, + off_dt_strings: u32, + off_mem_rsvmap: u32, + version: u32, + last_comp_version: u32, + boot_cpuid_phys: u32, + size_dt_strings: u32, + size_dt_struct: u32, +} + +struct FdtReserveEntry { + address: u64, + size: u64, +} + +enum FdtToken { + BeginNode = 1, + EndNode = 2, + Prop = 3, + Nop = 4, + End = 9, +} + +struct FdtTokenizer { + cur: *const u8, + end: *const u8, + strs: *const u8, +} + +const FDT_VERSION: u32 = 17; +const FDT_MAGIC: u32 = 0xd00dfeed; + +impl FdtTokenizer { + fn new(strs: *const u8, begin: *const u8, end: *const u8) -> Self { + Self { + cur: begin, + end, + strs, + } + } + + fn align(&mut self) { + self.cur = round_up(self.cur as usize, 4) as _; + } + + fn u32(&mut self) -> Option { + let next = self.cur.offset(mem::size_of::() as isize); + if next > self.end { + return None; + } + + let res = u32::from_be(*(self.cur as usize as *const u32)); + self.cur = next; + Some(res) + } + + fn token(&mut self) -> Option { + while let Some(v) = self.u32() { + if v != FdtToken::Nop as u32 { + return Some(v) + } + } + + None + } + + fn bytes(&mut self, size: usize) -> Option<*const u8> { + let next = self.cur.offset(size as isize); + + if next > self.end { + return None; + } + + let res = self.cur; + self.cur = next; + self.align(); + + Some(res) + } + + fn str(&mut self) -> Option<*const u8> { + let mut p = self.cur; + + while p < self.end { + if *p != 0 { + // Found the end of the string. + let res = self.cur; + self.cur = p.offset(1); + self.align(); + return Some(res) + } + } + + None + } + + fn next_property(&mut self, name: &mut *const u8, buf: &mut *const u8, size: &mut u32) -> Option<()> { + let token = self.token()?; + + if token != FdtToken::Prop as u32 { + // Rewind so that caller will get the same token. + self.cur = self.cur.offset(-(mem::size_of::() as isize)); + return None; + } + + let this = guard(self, |this| { + // Move cursor to the end so that caller won't get any new tokens. + this.cur = this.end; + }); + + *size = this.u32()?; + let nameoff = this.u32()?; + let buf = this.bytes(*size as usize)?; + + //TODO: Need to verify the strings. + *name = this.strs.offset(nameoff as isize); + + mem::forget(this); + Some(()) + } + + fn next_subnode(&mut self) -> Option<*const u8> { + let token = self.token()?; + if token != FdtToken::BeginNode as u32 { + // Rewind so that caller will get the same token. + self.cur = self.cur.offset(-(mem::size_of::() as isize)); + return None; + } + + match self.str() { + Some(name) => return Some(name), + None => { + // Move cursor to the end so that caller won't get any new + // tokens. + self.cur = self.end; + return None; + } + } + } + + fn skip_properties(&mut self) { + let mut name; + let mut buf; + let mut size; + while let Some(_) = self.next_property(&mut name, &mut buf, &mut size) {} + } + + fn skip_node(&mut self) -> Option<()> { + let mut pending = 1; + self.skip_properties(); + + while pending != 0 { + while let Some(_) = self.next_subnode() { + self.skip_properties(); + pending += 1; + } + + let token = self.token()?; + if token != FdtToken::EndNode as u32 { + self.cur = self.end; + return None; + } + + pending -= 1; + } + + Some(()) + } +} + +impl FdtNode { + pub fn new_root(hdr: &FdtHeader) -> Option { + let begin = u32::from_be(hdr.off_dt_struct); + let size = u32::from_be(hdr.size_dt_struct); + + // Check the magic number before anything else. + if hdr.magic != u32::from_be(FDT_MAGIC) { + return None; + } + + // Check the version. + let max_ver = u32::from_be(hdr.version); + let min_ver = u32::from_be(hdr.last_comp_version); + if FDT_VERSION < min_ver || FDT_VERSION > max_ver { + return None; + } + + let hdr_ptr = hdr as *const _ as usize as *const u8; + + // TODO: Verify that it is all within the fdt. + // TODO: Verify strings as well. + Some(Self { + hdr: ptr::null(), + begin: hdr_ptr.offset(begin as isize), + end: hdr_ptr.offset((begin + size) as isize), + strs: hdr_ptr.offset(u32::from_be(hdr.off_dt_strings) as isize), + }) + } + + pub unsafe fn read_property(&self, name: *const u8, buf: &mut *const u8, size: &mut u32) -> bool { + + let mut prop_name = ptr::null(); + let t = FdtTokenizer::new(self.strs, self.begin, self.end); + while let Some(_) = t.next_property(&mut prop_name, buf, size) { + if strcmp(prop_name, name) == 0 { + return true; + } + } + + false + } + + pub fn first_child(&mut self) -> Option<*const u8> { + let t = FdtTokenizer::new(self.strs, self.begin, self.end); + + t.skip_properties(); + + let child_name = t.next_subnode()?; + self.begin = t.cur; + + Some(child_name) + } + + pub fn next_sibling(&mut self) -> Option<*const u8> { + let t = FdtTokenizer::new(self.strs, self.begin, self.end); + + t.skip_node()?; + let sibling_name = t.next_subnode()?; + self.begin = t.cur; + + Some(sibling_name) + } + + pub unsafe fn find_child(&mut self, child: *const u8) -> Option<()> { + let t = FdtTokenizer::new(self.strs, self.begin, self.end); + t.skip_properties(); + + while let Some(name) = t.next_subnode() { + if strcmp(name, child) == 0 { + self.begin = t.cur; + return Some(()) + } + + t.skip_node(); + } + + None + } +} + +impl FdtHeader { + pub unsafe fn dump(&self) { + unsafe fn ascii_to_utf8(ptr: *const u8) -> &'static str { + let mut len = 0; + while *ptr.offset(len as isize) != 0u8 { len += 1; } + let bytes = slice::from_raw_parts(ptr, len); + str::from_utf8_unchecked(bytes) + } + + // Traverse the whole thing. + let node = match FdtNode::new_root(self) { + Some(node) => node, + None => { + dlog!("FDT failed validation.\n"); + return; + } + }; + + let t = FdtTokenizer::new(node.strs, node.begin, node.end); + let depth = 0; + + loop { + while let Some(name) = t.next_subnode() { + dlog!("{:1$}New node: \"{2}\"\n", "", 2 * depth, ascii_to_utf8(name)); + depth += 1; + let mut name = mem::uninitialized(); + let mut buf = mem::uninitialized(); + let mut size = mem::uninitialized(); + while let Some(_) = t.next_property(&mut name, &mut buf, &mut size) { + dlog!("{:1$}property: \"{2}\" (", "", 2 * depth, ascii_to_utf8(name)); + for i in 0..size { + dlog!("{}{:02x}", if i == 0 { "" } else { " " }, + *buf.offset(i as isize)); + } + dlog!(")\n"); + } + } + + let token = match t.token().filter(|t| *t != FdtToken::EndNode as u32) { + Some(token) => token, + None => return, + }; + + depth -= 1; + + if depth == 0 { + break; + } + } + + dlog!("fdt: off_mem_rsvmap={}\n", u32::from_be(self.off_mem_rsvmap)); + { + let e = self as *const _ as usize + u32::from_be(self.off_mem_rsvmap) as usize; + let entry = e as *const FdtReserveEntry; + + while (*entry).address != 0 || (*entry).size != 0 { + dlog!("Entry: {:p} (0x{:x} bytes)\n", u64::from_be((*entry).address) as *const u8, u64::from_be((*entry).size)); + entry = entry.offset(1); + } + } + } + + pub unsafe fn add_mem_reservation(&mut self, addr: u64, len: u64) { + // TODO: Clean this up. + let begin = (self as *const _ as usize as *const u8).offset(u32::from_be(self.off_mem_rsvmap) as isize); + let e = begin as *const FdtReserveEntry; + let old_size = u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap); + + self.totalsize = (u32::from_be(self.totalsize) + mem::size_of::() as u32).to_be(); + self.off_dt_struct = (u32::from_be(self.off_dt_struct) + mem::size_of::() as u32).to_be(); + self.off_dt_strings = (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); + + memmove_s(begin.offset(mem::size_of::() as isize) as usize as *mut _, + old_size as usize, begin as usize as *const _, old_size as usize); + (*e).address = addr.to_be(); + (*e).size = len.to_be(); + } + + pub fn total_size(&self) -> u32 { + u32::from_be(self.totalsize) + } +} diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs new file mode 100644 index 000000000..e61636e89 --- /dev/null +++ b/hfo2/src/fdt_handler.rs @@ -0,0 +1,108 @@ +/* + * Copyright 2019 Sanguk Park. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use core::mem; + +use crate::fdt::*; + +fn convert_number(data: *const u8, size: u32) -> u64 { + match size { + mem::size_of::() => u32::from_be(*(data as usize as *const u32)) as u64, + mem::size_of::() => u64::from_be(*(data as usize as *const u64)), + _ => 0, + } +} + +impl FdtNode { + fn read_number(&mut self, name: *const u8) -> Option { + let mut data = mem::uninitialized(); + let mut size = mem::uninitialized(); + + if !self.read_property(name, &mut data, &mut size) { + return false; + } + + match size { + mem::size_of::() | mem::size_of::() => { + Some(convert_number(data, size)) + } + _ => None, + } + } + + fn write_number(&mut self, name: *const u8, value: u64) -> bool { + let mut data = mem::uninitialized(); + let mut size = mem::uninitialized(); + + if !self.read_property(name, &mut data, &mut size) { + return false; + } + + match size { + mem::size_of::() => { + *(data as *mut u32) = u32::from_be(value); + } + mem::size_of::() => { + *(data as *mut u64) = u64::from_be(value); + } + _ => return false, + } + + true + } + + /// Finds the memory region where initrd is stored, and updates the fdt node + /// cursor to the node called "chosen". + fn find_initrd(&mut self, begin: &mut paddr_t, end: &mut paddr_t) -> bool { + if self.find_child("chosen\0").is_none() { + dlog!("Unable to find 'chosen'\n"); + return false; + } + + let initrd_begin = match self.read_number("linux,initrd-start\0") { + Some(initrd_begin) => initrd_begin, + None => { + dlog!("Unable to read linux,initrd-start\n"); + return false; + } + }; + + let initrd_end = match self.read_number("linux,initrd-end\0") { + Some(initrd_end) => initrd_end, + None => { + dlog!("Unable to read linux,initrd-end\n"); + return false; + } + }; + + *begin = pa_init(initrd_begin); + *end = pa_init(initrd_end); + + true + } + + fn find_cpus(&self, cpu_ids: &mut cpu_id_t, cpu_count: &mut usize) { + let node = self.clone(); + *cpu_count = 0; + + if node.find_child("cpus\0").is_none() { + dlog!("Unable to find 'cpus'\n"); + return; + } + + + } +} diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index b9116c01d..cf1d05f5f 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -46,6 +46,7 @@ mod api; mod arch; mod boot_params; mod cpu; +mod fdt; mod init; mod layout; mod load; @@ -54,6 +55,7 @@ mod mm; mod mpool; mod page; mod panic; +mod plat; mod slist; mod spci; mod spci_architected_message; diff --git a/hfo2/src/plat.rs b/hfo2/src/plat.rs new file mode 100644 index 000000000..6da6e31a1 --- /dev/null +++ b/hfo2/src/plat.rs @@ -0,0 +1,113 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::boot_params::*; +use crate::mm::*; +use crate::mpool::*; +use crate::addr::*; +use crate::layout::*; +use crate::arch::*; +use crate::fdt::*; + +/// Default implementation assumes the FDT has been linked into the image. +/// +/// This can be overridden e.g. to provide a fixed address or an address passed +/// by the loader. +fn plat_get_fdt_addr() -> paddr_t { + unsafe { layout_fdt_begin() } +} + +/// Default implementation assumes the initrd has been linked into the image. +/// +/// This can be overridden e.g. to provide a fixed address or an address passed +/// by the loader. +fn plat_get_initrd_range(stage1_locked: mm_stage1_locked, + begin: &mut paddr_t, end: &mut paddr_t, ppool: &mut MPool) { + *begin = unsafe { layout_initrd_begin() }; + *end = unsafe { layout_initrd_end() }; +} + +/// Default implementation assumes the FDT address is passed to the kernel. +/// +/// TODO: make this part of the VM configuration as secondary VMs will also need +/// to take arguments. +fn plat_get_kernel_arg() -> uintreg_t { + pa_addr(plat_get_fdt_addr()) as uintreg_t +} + + +/// Default implementation extracts the boot parameters from the FDT but the +/// initrd is provided separately. +#[no_mangle] +pub unsafe extern "C" fn plat_get_boot_params( + stage1_locked: mm_stage1_locked, + p: &mut BootParams, + ppool: &mut MPool, +) -> bool { + let mut ret = false; + let n; + + plat_get_initrd_range(stage1_locked, &mut p.initrd_begin, &mut p.initrd_end, ppool); + p.kernel_arg = plat_get_kernel_arg(); + + // Get the memory map from the FDT. + let fdt = fdt_map(stage1_locked, plat_get_fdt_addr(), &n, ppool); + if fdt.is_null() { + return false; + } + + if !fdt_find_child(&n, "") { + dlog!("Unable to find FDT root node.\n"); + // goto out_unmap_fdt; + if !fdt_unmap(stage1_locked, fdt, ppool) { + dlog!("Unable to unmap fdt."); + return false; + } + + return ret; + } + + fdt_find_cpus(&n, p.cpu_ids, p.cpu_count); + + p.mem_ranges_count = 0; + fdt_find_memory_ranges(&n, p); + + ret = true; + + // out_unmap_fdt: + if !fdt_unmap(stage1_locked, fdt, ppool) { + dlog!("Unable to unmap fdt."); + return false; + } + + return ret; +} + +/// Default implementation updates the FDT which is the argument passed to the +/// primary VM's kernel. +/// +/// TODO: in future, each VM will declare whether it expects an argument passed +/// and that will be static data e.g. it will provide its own FDT so there will +/// be no FDT modification. This is done because each VM has a very different +/// view of the system and we don't want to force VMs to require loader code +/// when another loader can load the data for it. +pub fn plat_update_boot_params( + stage1_locked: mm_stage1_locked, + p: &mut BootParamsUpdate, + ppool: &mut MPool, +) -> bool { + fdt_patch(stage1_locked, plat_get_fdt_addr(), p, ppool); +} From 864fa4ec55774ff48a2e27352095d6dad61fe570 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sun, 1 Sep 2019 04:44:51 +0900 Subject: [PATCH 05/45] Oxidize FDT (cont'd). --- hfo2/src/abi.rs | 2 +- hfo2/src/boot_params.rs | 13 -- hfo2/src/fdt.rs | 133 ++++++++++----- hfo2/src/fdt_handler.rs | 367 +++++++++++++++++++++++++++++++++++++--- hfo2/src/init.rs | 37 ++-- hfo2/src/lib.rs | 1 + hfo2/src/load.rs | 28 +-- hfo2/src/plat.rs | 48 +++--- 8 files changed, 491 insertions(+), 138 deletions(-) diff --git a/hfo2/src/abi.rs b/hfo2/src/abi.rs index e592eaf24..c3a7dbb5a 100644 --- a/hfo2/src/abi.rs +++ b/hfo2/src/abi.rs @@ -138,7 +138,7 @@ mod test { #[test] fn abi_hf_vcpu_run_return_encode_wait_for_interrupt() { let res = HfVCpuRunReturn::WaitForInterrupt { - ns: HF_SLEEP_INDEFINITE + ns: HF_SLEEP_INDEFINITE, }; assert_eq!(res.into_raw(), 0xffffffffffffff02); } diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs index 73d18514b..aaa219784 100644 --- a/hfo2/src/boot_params.rs +++ b/hfo2/src/boot_params.rs @@ -44,16 +44,3 @@ pub struct BootParamsUpdate { pub initrd_begin: paddr_t, pub initrd_end: paddr_t, } - -extern "C" { - pub fn plat_get_boot_params( - stage1_locked: mm_stage1_locked, - p: *mut BootParams, - ppool: *mut MPool, - ) -> bool; - pub fn plat_update_boot_params( - stage1_locked: mm_stage1_locked, - p: *mut BootParamsUpdate, - ppool: *mut MPool, - ) -> bool; -} diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 7f7018f0c..d8313ee33 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -19,8 +19,8 @@ use core::ptr; use core::slice; use core::str; -use crate::utils::*; use crate::std::*; +use crate::utils::*; use scopeguard::guard; @@ -37,7 +37,7 @@ pub struct FdtNode { } #[repr(C)] -struct FdtHeader { +pub struct FdtHeader { magic: u32, totalsize: u32, off_dt_struct: u32, @@ -85,7 +85,7 @@ impl FdtTokenizer { self.cur = round_up(self.cur as usize, 4) as _; } - fn u32(&mut self) -> Option { + unsafe fn u32(&mut self) -> Option { let next = self.cur.offset(mem::size_of::() as isize); if next > self.end { return None; @@ -96,17 +96,17 @@ impl FdtTokenizer { Some(res) } - fn token(&mut self) -> Option { + unsafe fn token(&mut self) -> Option { while let Some(v) = self.u32() { if v != FdtToken::Nop as u32 { - return Some(v) + return Some(v); } } None } - fn bytes(&mut self, size: usize) -> Option<*const u8> { + unsafe fn bytes(&mut self, size: usize) -> Option<*const u8> { let next = self.cur.offset(size as isize); if next > self.end { @@ -120,7 +120,7 @@ impl FdtTokenizer { Some(res) } - fn str(&mut self) -> Option<*const u8> { + unsafe fn str(&mut self) -> Option<*const u8> { let mut p = self.cur; while p < self.end { @@ -129,14 +129,19 @@ impl FdtTokenizer { let res = self.cur; self.cur = p.offset(1); self.align(); - return Some(res) + return Some(res); } } None } - fn next_property(&mut self, name: &mut *const u8, buf: &mut *const u8, size: &mut u32) -> Option<()> { + unsafe fn next_property( + &mut self, + name: &mut *const u8, + buf: &mut *const u8, + size: &mut u32, + ) -> Option<()> { let token = self.token()?; if token != FdtToken::Prop as u32 { @@ -145,7 +150,7 @@ impl FdtTokenizer { return None; } - let this = guard(self, |this| { + let mut this = guard(self, |this| { // Move cursor to the end so that caller won't get any new tokens. this.cur = this.end; }); @@ -161,7 +166,7 @@ impl FdtTokenizer { Some(()) } - fn next_subnode(&mut self) -> Option<*const u8> { + unsafe fn next_subnode(&mut self) -> Option<*const u8> { let token = self.token()?; if token != FdtToken::BeginNode as u32 { // Rewind so that caller will get the same token. @@ -172,7 +177,7 @@ impl FdtTokenizer { match self.str() { Some(name) => return Some(name), None => { - // Move cursor to the end so that caller won't get any new + // Move cursor to the end so that caller won't get any new // tokens. self.cur = self.end; return None; @@ -180,14 +185,14 @@ impl FdtTokenizer { } } - fn skip_properties(&mut self) { - let mut name; - let mut buf; - let mut size; + unsafe fn skip_properties(&mut self) { + let mut name = mem::uninitialized(); + let mut buf = mem::uninitialized(); + let mut size = mem::uninitialized(); while let Some(_) = self.next_property(&mut name, &mut buf, &mut size) {} } - fn skip_node(&mut self) -> Option<()> { + unsafe fn skip_node(&mut self) -> Option<()> { let mut pending = 1; self.skip_properties(); @@ -211,7 +216,7 @@ impl FdtTokenizer { } impl FdtNode { - pub fn new_root(hdr: &FdtHeader) -> Option { + pub unsafe fn new_root(hdr: &FdtHeader) -> Option { let begin = u32::from_be(hdr.off_dt_struct); let size = u32::from_be(hdr.size_dt_struct); @@ -239,10 +244,14 @@ impl FdtNode { }) } - pub unsafe fn read_property(&self, name: *const u8, buf: &mut *const u8, size: &mut u32) -> bool { - + pub unsafe fn read_property( + &self, + name: *const u8, + buf: &mut *const u8, + size: &mut u32, + ) -> bool { let mut prop_name = ptr::null(); - let t = FdtTokenizer::new(self.strs, self.begin, self.end); + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); while let Some(_) = t.next_property(&mut prop_name, buf, size) { if strcmp(prop_name, name) == 0 { return true; @@ -252,9 +261,9 @@ impl FdtNode { false } - pub fn first_child(&mut self) -> Option<*const u8> { - let t = FdtTokenizer::new(self.strs, self.begin, self.end); - + pub unsafe fn first_child(&mut self) -> Option<*const u8> { + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); + t.skip_properties(); let child_name = t.next_subnode()?; @@ -263,8 +272,8 @@ impl FdtNode { Some(child_name) } - pub fn next_sibling(&mut self) -> Option<*const u8> { - let t = FdtTokenizer::new(self.strs, self.begin, self.end); + pub unsafe fn next_sibling(&mut self) -> Option<*const u8> { + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); t.skip_node()?; let sibling_name = t.next_subnode()?; @@ -274,13 +283,13 @@ impl FdtNode { } pub unsafe fn find_child(&mut self, child: *const u8) -> Option<()> { - let t = FdtTokenizer::new(self.strs, self.begin, self.end); + let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); t.skip_properties(); while let Some(name) = t.next_subnode() { if strcmp(name, child) == 0 { self.begin = t.cur; - return Some(()) + return Some(()); } t.skip_node(); @@ -292,9 +301,11 @@ impl FdtNode { impl FdtHeader { pub unsafe fn dump(&self) { - unsafe fn ascii_to_utf8(ptr: *const u8) -> &'static str { + unsafe fn asciz_to_utf8(ptr: *const u8) -> &'static str { let mut len = 0; - while *ptr.offset(len as isize) != 0u8 { len += 1; } + while *ptr.offset(len as isize) != 0u8 { + len += 1; + } let bytes = slice::from_raw_parts(ptr, len); str::from_utf8_unchecked(bytes) } @@ -308,21 +319,34 @@ impl FdtHeader { } }; - let t = FdtTokenizer::new(node.strs, node.begin, node.end); - let depth = 0; + let mut t = FdtTokenizer::new(node.strs, node.begin, node.end); + let mut depth = 0; loop { while let Some(name) = t.next_subnode() { - dlog!("{:1$}New node: \"{2}\"\n", "", 2 * depth, ascii_to_utf8(name)); + dlog!( + "{:1$}New node: \"{2}\"\n", + "", + 2 * depth, + asciz_to_utf8(name) + ); depth += 1; let mut name = mem::uninitialized(); let mut buf = mem::uninitialized(); let mut size = mem::uninitialized(); while let Some(_) = t.next_property(&mut name, &mut buf, &mut size) { - dlog!("{:1$}property: \"{2}\" (", "", 2 * depth, ascii_to_utf8(name)); + dlog!( + "{:1$}property: \"{2}\" (", + "", + 2 * depth, + asciz_to_utf8(name) + ); for i in 0..size { - dlog!("{}{:02x}", if i == 0 { "" } else { " " }, - *buf.offset(i as isize)); + dlog!( + "{}{:02x}", + if i == 0 { "" } else { " " }, + *buf.offset(i as isize) + ); } dlog!(")\n"); } @@ -340,13 +364,20 @@ impl FdtHeader { } } - dlog!("fdt: off_mem_rsvmap={}\n", u32::from_be(self.off_mem_rsvmap)); + dlog!( + "fdt: off_mem_rsvmap={}\n", + u32::from_be(self.off_mem_rsvmap) + ); { let e = self as *const _ as usize + u32::from_be(self.off_mem_rsvmap) as usize; - let entry = e as *const FdtReserveEntry; + let mut entry = e as *const FdtReserveEntry; while (*entry).address != 0 || (*entry).size != 0 { - dlog!("Entry: {:p} (0x{:x} bytes)\n", u64::from_be((*entry).address) as *const u8, u64::from_be((*entry).size)); + dlog!( + "Entry: {:p} (0x{:x} bytes)\n", + u64::from_be((*entry).address) as *const u8, + u64::from_be((*entry).size) + ); entry = entry.offset(1); } } @@ -354,16 +385,24 @@ impl FdtHeader { pub unsafe fn add_mem_reservation(&mut self, addr: u64, len: u64) { // TODO: Clean this up. - let begin = (self as *const _ as usize as *const u8).offset(u32::from_be(self.off_mem_rsvmap) as isize); - let e = begin as *const FdtReserveEntry; + let begin = (self as *const _ as usize as *const u8) + .offset(u32::from_be(self.off_mem_rsvmap) as isize); + let e = begin as *mut FdtReserveEntry; let old_size = u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap); - self.totalsize = (u32::from_be(self.totalsize) + mem::size_of::() as u32).to_be(); - self.off_dt_struct = (u32::from_be(self.off_dt_struct) + mem::size_of::() as u32).to_be(); - self.off_dt_strings = (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); - - memmove_s(begin.offset(mem::size_of::() as isize) as usize as *mut _, - old_size as usize, begin as usize as *const _, old_size as usize); + self.totalsize = + (u32::from_be(self.totalsize) + mem::size_of::() as u32).to_be(); + self.off_dt_struct = + (u32::from_be(self.off_dt_struct) + mem::size_of::() as u32).to_be(); + self.off_dt_strings = + (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); + + memmove_s( + begin.offset(mem::size_of::() as isize) as usize as *mut _, + old_size as usize, + begin as usize as *const _, + old_size as usize, + ); (*e).address = addr.to_be(); (*e).size = len.to_be(); } diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index e61636e89..4bcdf175b 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -15,47 +15,57 @@ */ use core::mem; +use core::ptr::NonNull; +use crate::addr::*; +use crate::arch::*; +use crate::boot_params::*; use crate::fdt::*; +use crate::layout::*; +use crate::mm::*; +use crate::mpool::*; +use crate::page::*; +use crate::std::*; +use crate::types::*; -fn convert_number(data: *const u8, size: u32) -> u64 { +use scopeguard::{guard, ScopeGuard}; + +unsafe fn convert_number(data: *const u8, size: u32) -> u64 { match size { - mem::size_of::() => u32::from_be(*(data as usize as *const u32)) as u64, - mem::size_of::() => u64::from_be(*(data as usize as *const u64)), + 4 => u32::from_be(*(data as usize as *const u32)) as u64, + 8 => u64::from_be(*(data as usize as *const u64)), _ => 0, } } impl FdtNode { - fn read_number(&mut self, name: *const u8) -> Option { + unsafe fn read_number(&mut self, name: *const u8) -> Option { let mut data = mem::uninitialized(); let mut size = mem::uninitialized(); if !self.read_property(name, &mut data, &mut size) { - return false; + return None; } match size { - mem::size_of::() | mem::size_of::() => { - Some(convert_number(data, size)) - } + 4 | 8 => Some(convert_number(data, size)), _ => None, } } - fn write_number(&mut self, name: *const u8, value: u64) -> bool { + unsafe fn write_number(&mut self, name: *const u8, value: u64) -> bool { let mut data = mem::uninitialized(); let mut size = mem::uninitialized(); - + if !self.read_property(name, &mut data, &mut size) { return false; } match size { - mem::size_of::() => { - *(data as *mut u32) = u32::from_be(value); + 4 => { + *(data as *mut u32) = u32::from_be(value as u32); } - mem::size_of::() => { + 8 => { *(data as *mut u64) = u64::from_be(value); } _ => return false, @@ -63,16 +73,16 @@ impl FdtNode { true } - + /// Finds the memory region where initrd is stored, and updates the fdt node /// cursor to the node called "chosen". - fn find_initrd(&mut self, begin: &mut paddr_t, end: &mut paddr_t) -> bool { - if self.find_child("chosen\0").is_none() { + unsafe fn find_initrd(&mut self, begin: &mut paddr_t, end: &mut paddr_t) -> bool { + if self.find_child("chosen\0".as_ptr()).is_none() { dlog!("Unable to find 'chosen'\n"); return false; } - let initrd_begin = match self.read_number("linux,initrd-start\0") { + let initrd_begin = match self.read_number("linux,initrd-start\0".as_ptr()) { Some(initrd_begin) => initrd_begin, None => { dlog!("Unable to read linux,initrd-start\n"); @@ -80,7 +90,7 @@ impl FdtNode { } }; - let initrd_end = match self.read_number("linux,initrd-end\0") { + let initrd_end = match self.read_number("linux,initrd-end\0".as_ptr()) { Some(initrd_end) => initrd_end, None => { dlog!("Unable to read linux,initrd-end\n"); @@ -88,21 +98,328 @@ impl FdtNode { } }; - *begin = pa_init(initrd_begin); - *end = pa_init(initrd_end); + *begin = pa_init(initrd_begin as usize); + *end = pa_init(initrd_end as usize); true } - fn find_cpus(&self, cpu_ids: &mut cpu_id_t, cpu_count: &mut usize) { - let node = self.clone(); + pub unsafe fn find_cpus(&self, cpu_ids: *mut cpu_id_t, cpu_count: &mut usize) -> Option<()> { + let mut node = self.clone(); *cpu_count = 0; - if node.find_child("cpus\0").is_none() { + node.find_child("cpus\0".as_ptr()).or_else(|| { dlog!("Unable to find 'cpus'\n"); - return; + None + })?; + + let address_size = node + .read_number("#address-cells\0".as_ptr()) + .map(|size| size as usize * mem::size_of::()) + .unwrap_or(mem::size_of::()); + + let mut name = node.first_child()?; + + loop { + let mut data = mem::uninitialized(); + let mut size = mem::uninitialized(); + + if !node.read_property("device_type\0".as_ptr(), &mut data, &mut size) + || size != "cpu\0".len() as u32 + || memcmp_rs( + data as usize as *const _, + "cpu\0".as_ptr() as usize as *const _, + "cpu\0".len(), + ) != 0 + || !node.read_property("reg".as_ptr(), &mut data, &mut size) + { + continue; + } + + // Get all entries for this CPU. + while size as usize >= address_size { + if *cpu_count >= MAX_CPUS { + dlog!("Found more than {} CPUs\n", MAX_CPUS); + return None; + } + + *cpu_ids.offset(*cpu_count as isize) = + convert_number(data, address_size as u32) as cpu_id_t; + *cpu_count += 1; + + size -= address_size as u32; + data = data.offset(address_size as isize); + } + + if let Some(sibling_name) = node.next_sibling() { + name = sibling_name; + } else { + break; + } + } + + Some(()) + } + + pub unsafe fn find_memory_ranges(&self, p: &mut BootParams) -> Option<()> { + let mut node = self.clone(); + + // Get the sizes of memory range addresses and sizes. + let address_size = node + .read_number("#address-cells\0".as_ptr()) + .map(|size| size as usize * mem::size_of::()) + .unwrap_or(mem::size_of::()); + + let size_size = node + .read_number("#size-cells\0".as_ptr()) + .map(|size| size as usize * mem::size_of::()) + .unwrap_or(mem::size_of::()); + + let entry_size = address_size + size_size; + + // Look for nodes with the device_type set to "memory". + let mut name = node.first_child()?; + let mut mem_range_index = 0; + + loop { + let mut data = mem::uninitialized(); + let mut size = mem::uninitialized(); + + if !node.read_property("device_type\0".as_ptr(), &mut data, &mut size) + || size as usize != "memory\0".len() + || memcmp_rs( + data as usize as *const _, + "memory\0".as_ptr() as usize as *const _, + "memory\0".len(), + ) != 0 + || !node.read_property("reg\0".as_ptr(), &mut data, &mut size) + { + continue; + } + + // Traverse all memory ranges within this node. + while size as usize >= entry_size { + let addr = convert_number(data, address_size as u32) as usize; + let len = + convert_number(data.offset(address_size as isize), size_size as u32) as usize; + + if mem_range_index < MAX_MEM_RANGES { + p.mem_ranges[mem_range_index].begin = pa_init(addr); + p.mem_ranges[mem_range_index].end = pa_init(addr + len); + + mem_range_index += 1; + } else { + dlog!("Found memory range {} in FDT but only {} supported, ignoring additional range of size {}.\n", mem_range_index, MAX_MEM_RANGES, len); + } + + size -= entry_size as u32; + data = data.offset(entry_size as isize); + } + + if let Some(sibling_name) = node.next_sibling() { + name = sibling_name; + } else { + break; + } } - + p.mem_ranges_count = mem_range_index; + Some(()) } } + +pub unsafe fn fdt_map( + stage1_ptable: &mut PageTable, + fdt_addr: paddr_t, + node: &mut FdtNode, + ppool: &mut MPool, +) -> Option> { + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + Mode::R, + ppool, + ) + .is_err() + { + dlog!("Unable to map FDT header.\n"); + return None; + } + + let mut stage1_ptable = guard(stage1_ptable, |ptable| { + let _ = ptable.unmap( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + ppool, + ); + }); + + let fdt = pa_addr(fdt_addr) as *mut FdtHeader; + + if let Some(root) = FdtNode::new_root(&*fdt) { + *node = root; + } else { + dlog!("FDT failed validation.\n"); + return None; + } + + // Map the rest of the fdt in. + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, (*fdt).total_size() as usize), + Mode::R, + ppool, + ) + .is_err() + { + dlog!("Unable to map full FDT.\n"); + return None; + } + + mem::forget(stage1_ptable); + NonNull::new(fdt) +} + +pub unsafe fn fdt_unmap( + stage1_ptable: &mut PageTable, + fdt: *mut FdtHeader, + ppool: &mut MPool, +) -> Result<(), ()> { + let fdt_addr = pa_from_va(va_from_ptr(fdt as usize as *const _)); + + stage1_ptable.unmap( + fdt_addr, + pa_add(fdt_addr, (*fdt).total_size() as usize), + ppool, + ) +} + +pub unsafe fn fdt_patch( + stage1_ptable: &mut PageTable, + fdt_addr: paddr_t, + p: &mut BootParamsUpdate, + ppool: &mut MPool, +) -> Result<(), ()> { + // Map the fdt header in. + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + Mode::R, + ppool, + ) + .is_err() + { + dlog!("Unable to map FDT header.\n"); + return Err(()); + } + + let mut stage1_ptable = guard(stage1_ptable, |ptable| { + let _ = ptable.unmap( + fdt_addr, + pa_add(fdt_addr, mem::size_of::()), + ppool, + ); + }); + + let fdt = pa_addr(fdt_addr) as *mut FdtHeader; + + let mut node = FdtNode::new_root(&*fdt) + .or_else(|| { + dlog!("FDT failed validation.\n"); + None + }) + .ok_or(())?; + let total_size = (*fdt).total_size(); + + // Map the fdt (+ a page) in r/w mode in preparation for updating it. + if stage1_ptable + .identity_map( + fdt_addr, + pa_add(fdt_addr, total_size as usize + PAGE_SIZE), + Mode::R | Mode::W, + ppool, + ) + .is_err() + { + dlog!("Unable to map FDT in r/w mode.\n"); + return Err(()); + } + + let stage1_ptable = guard(ScopeGuard::into_inner(stage1_ptable), |ptable| { + if ptable + .unmap( + fdt_addr, + pa_add(fdt_addr, total_size as usize + PAGE_SIZE), + ppool, + ) + .is_err() + { + dlog!("Unable to unmap writable FDT.\n"); + } + }); + + if node.find_child("\0".as_ptr()).is_none() { + dlog!("Unable to find FDT root node.\n"); + return Err(()); + } + + if node.find_child("chosen\0".as_ptr()).is_none() { + dlog!("Unable to find 'chosen'\n"); + return Err(()); + } + + // Patch FDT to point to new ramdisk. + if !node.write_number( + "linux,initrd-start\0".as_ptr(), + pa_addr(p.initrd_begin) as u64, + ) { + dlog!("Unable to write linux,initrd-start\n"); + return Err(()); + } + + if !node.write_number("linux,initrd-end\0".as_ptr(), pa_addr(p.initrd_end) as u64) { + dlog!("Unable to write linux,initrd-end\n"); + return Err(()); + } + + // Patch FDT to reserve hypervisor memory so the primary VM doesn't try to + // use it. + (*fdt).add_mem_reservation( + pa_addr(layout_text_begin()) as u64, + pa_difference(layout_text_begin(), layout_text_end()) as u64, + ); + (*fdt).add_mem_reservation( + pa_addr(layout_rodata_begin()) as u64, + pa_difference(layout_rodata_begin(), layout_rodata_end()) as u64, + ); + (*fdt).add_mem_reservation( + pa_addr(layout_data_begin()) as u64, + pa_difference(layout_data_begin(), layout_data_end()) as u64, + ); + + // Patch FDT to reserve memory for secondary VMs. + for i in 0..p.reserved_ranges_count { + (*fdt).add_mem_reservation( + pa_addr(p.reserved_ranges[i].begin) as u64, + pa_addr(p.reserved_ranges[i].end) as u64 - pa_addr(p.reserved_ranges[i].begin) as u64, + ); + } + + let stage1_ptable = ScopeGuard::into_inner(stage1_ptable); + if stage1_ptable + .unmap( + fdt_addr, + pa_add(fdt_addr, (*fdt).total_size() as usize + PAGE_SIZE), + ppool, + ) + .is_err() + { + dlog!("Unable to unmap writable FDT.\n"); + return Err(()); + } + + Ok(()) +} diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index d099c28ca..89135b585 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -25,6 +25,7 @@ use crate::memiter::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; +use crate::plat::*; use crate::types::*; use crate::vm::*; @@ -41,7 +42,6 @@ static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit( /// Performs one-time initialisation of the hypervisor. unsafe fn one_time_init() { - // Make sure the console is initialised before calling dlog. plat_console_init(); @@ -64,10 +64,10 @@ unsafe fn one_time_init() { dlog_enable_lock(); mpool_enable_locks(); - let mut mm_stage1_locked = mm_lock_stage1(); + let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); let mut params = mem::uninitialized(); - if !plat_get_boot_params(mm_stage1_locked, &mut params, &mut ppool) { + if !plat_get_boot_params(&mut hypervisor_ptable, &mut params, &mut ppool) { panic!("unable to retrieve boot params"); } @@ -88,21 +88,19 @@ unsafe fn one_time_init() { ); // Map initrd in, and initialise cpio parser. - let initrd = mm_identity_map( - mm_stage1_locked, - params.initrd_begin, - params.initrd_end, - Mode::R, - &ppool, - ); - if initrd.is_null() { + if hypervisor_ptable + .identity_map(params.initrd_begin, params.initrd_end, Mode::R, &ppool) + .is_err() + { panic!("unable to map initrd in"); } + let initrd = pa_addr(params.initrd_begin) as *mut _; + let mut cpio = mem::uninitialized(); memiter_init( &mut cpio, - initrd as *mut _ as usize as *mut _, + initrd, pa_difference(params.initrd_begin, params.initrd_end), ); @@ -110,7 +108,7 @@ unsafe fn one_time_init() { // Load all VMs. if !load_primary( - mm_stage1_locked, + &mut hypervisor_ptable, &cpio, params.kernel_arg, &mut primary_initrd, @@ -126,17 +124,22 @@ unsafe fn one_time_init() { update.initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next as usize as *const _)); update.initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)); update.reserved_ranges_count = 0; - if !load_secondary(mm_stage1_locked, &cpio, ¶ms, &mut update, &mut ppool) { + if !load_secondary( + &mut hypervisor_ptable, + &cpio, + ¶ms, + &mut update, + &mut ppool, + ) { panic!("unable to load secondary VMs"); } // Prepare to run by updating bootparams as seen by primary VM. - if !plat_update_boot_params(mm_stage1_locked, &mut update, &mut ppool) { + if !plat_update_boot_params(&mut hypervisor_ptable, &mut update, &mut ppool) { panic!("plat_update_boot_params failed"); } - mm_defrag(mm_stage1_locked, &ppool); - mm_unlock_stage1(&mut mm_stage1_locked); + hypervisor_ptable.defrag(&ppool); // Initialise the API page pool. ppool will be empty from now on. api_init(&ppool); diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index cf1d05f5f..efa957fee 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -47,6 +47,7 @@ mod arch; mod boot_params; mod cpu; mod fdt; +mod fdt_handler; mod init; mod layout; mod load; diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 05f53db95..e7d006b61 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -40,34 +40,36 @@ use crate::vm::*; /// disabled. When switching to the partitions, the caching is initially /// disabled so the data must be available without the cache. unsafe fn copy_to_unmapped( - stage1_locked: mm_stage1_locked, + hypervisor_ptable: &mut PageTable, to: paddr_t, from: *const c_void, size: usize, - ppool: *mut MPool, + ppool: &mut MPool, ) -> bool { let to_end = pa_add(to, size); - let ptr = mm_identity_map(stage1_locked, to, to_end, Mode::W, ppool); - if ptr.is_null() { + if hypervisor_ptable + .identity_map(to, to_end, Mode::W, ppool) + .is_err() + { return false; } - memcpy_s(ptr as usize as *mut _, size, from, size); - arch_mm_write_back_dcache(ptr as usize, size); + memcpy_s(pa_addr(to) as *mut _, size, from, size); + arch_mm_write_back_dcache(pa_addr(to), size); - mm_unmap(stage1_locked, to, to_end, ppool); + hypervisor_ptable.unmap(to, to_end, ppool); true } /// Loads the primary VM. pub unsafe fn load_primary( - stage1_locked: mm_stage1_locked, + hypervisor_ptable: &mut PageTable, cpio: *const MemIter, kernel_arg: uintreg_t, initrd: *mut MemIter, - ppool: *mut MPool, + ppool: &mut MPool, ) -> bool { let mut it = mem::uninitialized(); let primary_begin = layout_primary_begin(); @@ -82,7 +84,7 @@ pub unsafe fn load_primary( pa_addr(primary_begin) as *const u8 ); if !copy_to_unmapped( - stage1_locked, + hypervisor_ptable, primary_begin, it.next as usize as *mut _, it.limit.offset_from(it.next) as usize, @@ -198,11 +200,11 @@ unsafe fn update_reserved_ranges( /// Loads all secondary VMs into the memory ranges from the given params. /// Memory reserved for the VMs is added to the `reserved_ranges` of `update`. pub unsafe fn load_secondary( - stage1_locked: mm_stage1_locked, + hypervisor_ptable: &mut PageTable, cpio: *const MemIter, params: *const BootParams, update: *mut BootParamsUpdate, - ppool: *mut MPool, + ppool: &mut MPool, ) -> bool { let mut mem_ranges_available: [MemRange; MAX_MEM_RANGES] = mem::uninitialized(); // static_assert( @@ -272,7 +274,7 @@ pub unsafe fn load_secondary( } if !copy_to_unmapped( - stage1_locked, + hypervisor_ptable, secondary_mem_begin, kernel.next as usize as *const _, kernel.limit.offset_from(kernel.next) as usize, diff --git a/hfo2/src/plat.rs b/hfo2/src/plat.rs index 6da6e31a1..a35269b8a 100644 --- a/hfo2/src/plat.rs +++ b/hfo2/src/plat.rs @@ -14,13 +14,19 @@ * limitations under the License. */ -use crate::boot_params::*; -use crate::mm::*; -use crate::mpool::*; +//! TODO(HfO2): Functions here were denoted by `#pragma weak`, but no other +//! definitions are in Hafnium C code. How can I denote this? + +use core::mem; + use crate::addr::*; -use crate::layout::*; use crate::arch::*; +use crate::boot_params::*; use crate::fdt::*; +use crate::fdt_handler::*; +use crate::layout::*; +use crate::mm::*; +use crate::mpool::*; /// Default implementation assumes the FDT has been linked into the image. /// @@ -34,8 +40,7 @@ fn plat_get_fdt_addr() -> paddr_t { /// /// This can be overridden e.g. to provide a fixed address or an address passed /// by the loader. -fn plat_get_initrd_range(stage1_locked: mm_stage1_locked, - begin: &mut paddr_t, end: &mut paddr_t, ppool: &mut MPool) { +fn plat_get_initrd_range(begin: &mut paddr_t, end: &mut paddr_t) { *begin = unsafe { layout_initrd_begin() }; *end = unsafe { layout_initrd_end() }; } @@ -48,31 +53,30 @@ fn plat_get_kernel_arg() -> uintreg_t { pa_addr(plat_get_fdt_addr()) as uintreg_t } - /// Default implementation extracts the boot parameters from the FDT but the /// initrd is provided separately. -#[no_mangle] pub unsafe extern "C" fn plat_get_boot_params( - stage1_locked: mm_stage1_locked, + ptable: &mut PageTable, p: &mut BootParams, ppool: &mut MPool, ) -> bool { let mut ret = false; - let n; + let mut n = mem::uninitialized(); - plat_get_initrd_range(stage1_locked, &mut p.initrd_begin, &mut p.initrd_end, ppool); + plat_get_initrd_range(&mut p.initrd_begin, &mut p.initrd_end); p.kernel_arg = plat_get_kernel_arg(); // Get the memory map from the FDT. - let fdt = fdt_map(stage1_locked, plat_get_fdt_addr(), &n, ppool); - if fdt.is_null() { + let fdt = fdt_map(ptable, plat_get_fdt_addr(), &mut n, ppool); + if fdt.is_none() { return false; } + let fdt = fdt.unwrap().as_ptr(); - if !fdt_find_child(&n, "") { + if n.find_child("\0".as_ptr()).is_none() { dlog!("Unable to find FDT root node.\n"); // goto out_unmap_fdt; - if !fdt_unmap(stage1_locked, fdt, ppool) { + if fdt_unmap(ptable, fdt, ppool).is_err() { dlog!("Unable to unmap fdt."); return false; } @@ -80,15 +84,15 @@ pub unsafe extern "C" fn plat_get_boot_params( return ret; } - fdt_find_cpus(&n, p.cpu_ids, p.cpu_count); + n.find_cpus(p.cpu_ids.as_mut_ptr(), &mut p.cpu_count); p.mem_ranges_count = 0; - fdt_find_memory_ranges(&n, p); + n.find_memory_ranges(p); ret = true; // out_unmap_fdt: - if !fdt_unmap(stage1_locked, fdt, ppool) { + if fdt_unmap(ptable, fdt, ppool).is_err() { dlog!("Unable to unmap fdt."); return false; } @@ -102,12 +106,12 @@ pub unsafe extern "C" fn plat_get_boot_params( /// TODO: in future, each VM will declare whether it expects an argument passed /// and that will be static data e.g. it will provide its own FDT so there will /// be no FDT modification. This is done because each VM has a very different -/// view of the system and we don't want to force VMs to require loader code +/// view of the system and we don't want to force VMs to require loader code /// when another loader can load the data for it. -pub fn plat_update_boot_params( - stage1_locked: mm_stage1_locked, +pub unsafe fn plat_update_boot_params( + ptable: &mut PageTable, p: &mut BootParamsUpdate, ppool: &mut MPool, ) -> bool { - fdt_patch(stage1_locked, plat_get_fdt_addr(), p, ppool); + fdt_patch(ptable, plat_get_fdt_addr(), p, ppool).is_ok() } From 8a640d834ba2e55d7aedd9b9d1e1eab5194ce990 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sun, 1 Sep 2019 23:55:12 +0900 Subject: [PATCH 06/45] Export C functions. --- hfo2/src/fdt.rs | 55 ++++++ hfo2/src/fdt_handler.rs | 43 +++- src/BUILD.gn | 21 -- src/fdt.c | 426 ---------------------------------------- src/fdt_handler.c | 385 ------------------------------------ 5 files changed, 94 insertions(+), 836 deletions(-) delete mode 100644 src/fdt.c delete mode 100644 src/fdt_handler.c diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index d8313ee33..f8c0951c2 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -411,3 +411,58 @@ impl FdtHeader { u32::from_be(self.totalsize) } } + +#[no_mangle] +pub extern "C" fn fdt_header_size() -> usize { + mem::size_of::() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_total_size(hdr: *mut FdtHeader) -> u32 { + (*hdr).total_size() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_dump(hdr: *mut FdtHeader) { + (*hdr).dump() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_root_node(node: *mut FdtNode, hdr: *const FdtHeader) -> bool { + FdtNode::new_root(&*hdr) + .map_or(false, |n| { + ptr::write(node, n); + true + }) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_child(node: *mut FdtNode, child: *const u8) -> bool { + (*node).find_child(child).is_some() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_first_child(node: *mut FdtNode, child_name: *mut *const u8) -> bool { + (*node).first_child().map_or(false, |name| { + ptr::write(child_name, name); + true + }) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_next_sibling(node: *mut FdtNode, sibling_name: *mut *const u8) -> bool { + (*node).next_sibling().map_or(false, |name| { + ptr::write(sibling_name, name); + true + }) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_read_property(node: *mut FdtNode, name: *const u8, buf: *mut *const u8, size: *mut u32) -> bool { + (*node).read_property(name, &mut *buf, &mut *size) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_add_mem_reserveation(hdr: *mut FdtHeader, addr: u64, len: u64) { + (*hdr).add_mem_reservation(addr, len) +} diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index 4bcdf175b..219a3cb67 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -15,7 +15,7 @@ */ use core::mem; -use core::ptr::NonNull; +use core::ptr::{self, NonNull}; use crate::addr::*; use crate::arch::*; @@ -228,7 +228,7 @@ impl FdtNode { } } -pub unsafe fn fdt_map( +pub unsafe fn map( stage1_ptable: &mut PageTable, fdt_addr: paddr_t, node: &mut FdtNode, @@ -282,7 +282,7 @@ pub unsafe fn fdt_map( NonNull::new(fdt) } -pub unsafe fn fdt_unmap( +pub unsafe fn unmap( stage1_ptable: &mut PageTable, fdt: *mut FdtHeader, ppool: &mut MPool, @@ -296,7 +296,7 @@ pub unsafe fn fdt_unmap( ) } -pub unsafe fn fdt_patch( +pub unsafe fn patch( stage1_ptable: &mut PageTable, fdt_addr: paddr_t, p: &mut BootParamsUpdate, @@ -423,3 +423,38 @@ pub unsafe fn fdt_patch( Ok(()) } + +#[no_mangle] +pub unsafe extern "C" fn fdt_map(stage1_locked: mm_stage1_locked, + fdt_addr: paddr_t, n: *mut FdtNode, ppool: *mut MPool) -> *mut FdtHeader { + match map(&mut stage1_locked, fdt_addr, &mut *n, &mut *ppool) { + Some(ret) => ret.as_ptr(), + None => ptr::null_mut(), + } +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_unmap(stage1_locked: mm_stage1_locked, + fdt: *mut FdtHeader, ppool: *mut MPool) -> bool { + unmap(&mut stage1_locked, &mut *fdt, &mut *ppool).is_ok() +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_cpus(root: *const FdtNode, cpu_ids: *mut cpu_id_t, cpu_count: *mut usize) { + (*root).find_cpus(cpu_ids, &mut *cpu_count); +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_memory_ranges(root: *const FdtNode, p: *mut BootParams) { + (*root).find_memory_ranges(&mut *p); +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_find_initrd(n: *mut FdtNode, begin: *mut paddr_t, end: *mut paddr_t) -> bool { + (*n).find_initrd(&mut *begin, &mut *end) +} + +#[no_mangle] +pub unsafe extern "C" fn fdt_patch(stage1_locked: mm_stage1_locked, fdt_addr: paddr_t, p: *mut BootParamsUpdate, ppool: *mut MPool) -> bool { + patch(&mut stage1_locked, fdt_addr, &mut *p, &mut *ppool).is_ok() +} diff --git a/src/BUILD.gn b/src/BUILD.gn index 22ce009bf..a2cf268b0 100644 --- a/src/BUILD.gn +++ b/src/BUILD.gn @@ -83,31 +83,10 @@ source_set("dlog") { } source_set("fdt_handler") { - sources = [ - "fdt_handler.c", - ] - deps = [ - ":fdt", - ] - - if (is_debug) { - deps += [ ":dlog" ] - } } # Flattened Device Tree (FDT) utilities. source_set("fdt") { - sources = [ - "fdt.c", - ] - - deps = [ - ":std", - ] - - if (is_debug) { - deps += [ ":dlog" ] - } } source_set("memiter") { diff --git a/src/fdt.c b/src/fdt.c deleted file mode 100644 index de54189f4..000000000 --- a/src/fdt.c +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hf/fdt.h" - -#include - -#include "hf/dlog.h" -#include "hf/std.h" - -struct fdt_header { - uint32_t magic; - uint32_t totalsize; - uint32_t off_dt_struct; - uint32_t off_dt_strings; - uint32_t off_mem_rsvmap; - uint32_t version; - uint32_t last_comp_version; - uint32_t boot_cpuid_phys; - uint32_t size_dt_strings; - uint32_t size_dt_struct; -}; - -struct fdt_reserve_entry { - uint64_t address; - uint64_t size; -}; - -enum fdt_token { - FDT_BEGIN_NODE = 1, - FDT_END_NODE = 2, - FDT_PROP = 3, - FDT_NOP = 4, - FDT_END = 9, -}; - -struct fdt_tokenizer { - const char *cur; - const char *end; - const char *strs; -}; - -#define FDT_VERSION 17 -#define FDT_MAGIC 0xd00dfeed - -static void fdt_tokenizer_init(struct fdt_tokenizer *t, const char *strs, - const char *begin, const char *end) -{ - t->strs = strs; - t->cur = begin; - t->end = end; -} - -static void fdt_tokenizer_align(struct fdt_tokenizer *t) -{ - t->cur = (char *)align_up(t->cur, 4); -} - -static bool fdt_tokenizer_uint32(struct fdt_tokenizer *t, uint32_t *res) -{ - const char *next = t->cur + sizeof(*res); - - if (next > t->end) { - return false; - } - - *res = be32toh(*(uint32_t *)t->cur); - t->cur = next; - - return true; -} - -static bool fdt_tokenizer_token(struct fdt_tokenizer *t, uint32_t *res) -{ - uint32_t v; - - while (fdt_tokenizer_uint32(t, &v)) { - if (v != FDT_NOP) { - *res = v; - return true; - } - } - return false; -} - -static bool fdt_tokenizer_bytes(struct fdt_tokenizer *t, const char **res, - size_t size) -{ - const char *next = t->cur + size; - - if (next > t->end) { - return false; - } - - *res = t->cur; - t->cur = next; - fdt_tokenizer_align(t); - - return true; -} - -static bool fdt_tokenizer_str(struct fdt_tokenizer *t, const char **res) -{ - const char *p; - - for (p = t->cur; p < t->end; p++) { - if (!*p) { - /* Found the end of the string. */ - *res = t->cur; - t->cur = p + 1; - fdt_tokenizer_align(t); - return true; - } - } - - return false; -} - -bool fdt_root_node(struct fdt_node *node, const struct fdt_header *hdr) -{ - uint32_t max_ver; - uint32_t min_ver; - uint32_t begin = be32toh(hdr->off_dt_struct); - uint32_t size = be32toh(hdr->size_dt_struct); - - memset_s(node, sizeof(*node), 0, sizeof(*node)); - - /* Check the magic number before anything else. */ - if (hdr->magic != be32toh(FDT_MAGIC)) { - return false; - } - - /* Check the version. */ - max_ver = be32toh(hdr->version); - min_ver = be32toh(hdr->last_comp_version); - if (FDT_VERSION < min_ver || FDT_VERSION > max_ver) { - return false; - } - - /* TODO: Verify that it is all within the fdt. */ - node->begin = (const char *)hdr + begin; - node->end = node->begin + size; - - /* TODO: Verify strings as well. */ - node->strs = (char *)hdr + be32toh(hdr->off_dt_strings); - - return true; -} - -static bool fdt_next_property(struct fdt_tokenizer *t, const char **name, - const char **buf, uint32_t *size) -{ - uint32_t token; - uint32_t nameoff; - - if (!fdt_tokenizer_token(t, &token)) { - return false; - } - - if (token != FDT_PROP) { - /* Rewind so that caller will get the same token. */ - t->cur -= sizeof(uint32_t); - return false; - } - - if (!fdt_tokenizer_uint32(t, size) || - !fdt_tokenizer_uint32(t, &nameoff) || - !fdt_tokenizer_bytes(t, buf, *size)) { - /* - * Move cursor to the end so that caller won't get any new - * tokens. - */ - t->cur = t->end; - return false; - } - - /* TODO: Need to verify the strings. */ - *name = t->strs + nameoff; - - return true; -} - -static bool fdt_next_subnode(struct fdt_tokenizer *t, const char **name) -{ - uint32_t token; - - if (!fdt_tokenizer_token(t, &token)) { - return false; - } - - if (token != FDT_BEGIN_NODE) { - /* Rewind so that caller will get the same token. */ - t->cur -= sizeof(uint32_t); - return false; - } - - if (!fdt_tokenizer_str(t, name)) { - /* - * Move cursor to the end so that caller won't get any new - * tokens. - */ - t->cur = t->end; - return false; - } - - return true; -} - -static void fdt_skip_properties(struct fdt_tokenizer *t) -{ - const char *name; - const char *buf; - uint32_t size; - - while (fdt_next_property(t, &name, &buf, &size)) { - /* do nothing */ - } -} - -static bool fdt_skip_node(struct fdt_tokenizer *t) -{ - const char *name; - uint32_t token; - size_t pending = 1; - - fdt_skip_properties(t); - - do { - while (fdt_next_subnode(t, &name)) { - fdt_skip_properties(t); - pending++; - } - - if (!fdt_tokenizer_token(t, &token)) { - return false; - } - - if (token != FDT_END_NODE) { - t->cur = t->end; - return false; - } - - pending--; - } while (pending); - - return true; -} - -bool fdt_read_property(const struct fdt_node *node, const char *name, - const char **buf, uint32_t *size) -{ - struct fdt_tokenizer t; - const char *prop_name; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - while (fdt_next_property(&t, &prop_name, buf, size)) { - if (!strcmp(prop_name, name)) { - return true; - } - } - - return false; -} - -bool fdt_first_child(struct fdt_node *node, const char **child_name) -{ - struct fdt_tokenizer t; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - fdt_skip_properties(&t); - - if (!fdt_next_subnode(&t, child_name)) { - return false; - } - - node->begin = t.cur; - - return true; -} - -bool fdt_next_sibling(struct fdt_node *node, const char **sibling_name) -{ - struct fdt_tokenizer t; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - if (!fdt_skip_node(&t)) { - return false; - } - - if (!fdt_next_subnode(&t, sibling_name)) { - return false; - } - - node->begin = t.cur; - - return true; -} - -bool fdt_find_child(struct fdt_node *node, const char *child) -{ - struct fdt_tokenizer t; - const char *name; - - fdt_tokenizer_init(&t, node->strs, node->begin, node->end); - - fdt_skip_properties(&t); - - while (fdt_next_subnode(&t, &name)) { - if (!strcmp(name, child)) { - node->begin = t.cur; - return true; - } - - fdt_skip_node(&t); - } - - return false; -} - -void fdt_dump(struct fdt_header *hdr) -{ - uint32_t token; - size_t depth = 0; - const char *name; - struct fdt_tokenizer t; - struct fdt_node node; - - /* Traverse the whole thing. */ - if (!fdt_root_node(&node, hdr)) { - dlog("FDT failed validation.\n"); - return; - } - - fdt_tokenizer_init(&t, node.strs, node.begin, node.end); - - do { - while (fdt_next_subnode(&t, &name)) { - const char *buf; - uint32_t size; - - dlog("%*sNew node: \"%s\"\n", 2 * depth, "", name); - depth++; - while (fdt_next_property(&t, &name, &buf, &size)) { - uint32_t i; - - dlog("%*sproperty: \"%s\" (", 2 * depth, "", - name); - for (i = 0; i < size; i++) { - dlog("%s%02x", i == 0 ? "" : " ", - buf[i]); - } - dlog(")\n"); - } - } - - if (!fdt_tokenizer_token(&t, &token)) { - return; - } - - if (token != FDT_END_NODE) { - return; - } - - depth--; - } while (depth); - - dlog("fdt: off_mem_rsvmap=%u\n", be32toh(hdr->off_mem_rsvmap)); - { - struct fdt_reserve_entry *e = - (struct fdt_reserve_entry - *)((uintptr_t)hdr + - be32toh(hdr->off_mem_rsvmap)); - while (e->address || e->size) { - dlog("Entry: %p (0x%x bytes)\n", be64toh(e->address), - be64toh(e->size)); - e++; - } - } -} - -void fdt_add_mem_reservation(struct fdt_header *hdr, uint64_t addr, - uint64_t len) -{ - /* TODO: Clean this up. */ - uint8_t *begin = (uint8_t *)hdr + be32toh(hdr->off_mem_rsvmap); - struct fdt_reserve_entry *e = (struct fdt_reserve_entry *)begin; - size_t old_size = - be32toh(hdr->totalsize) - be32toh(hdr->off_mem_rsvmap); - - hdr->totalsize = htobe32(be32toh(hdr->totalsize) + - sizeof(struct fdt_reserve_entry)); - hdr->off_dt_struct = htobe32(be32toh(hdr->off_dt_struct) + - sizeof(struct fdt_reserve_entry)); - hdr->off_dt_strings = htobe32(be32toh(hdr->off_dt_strings) + - sizeof(struct fdt_reserve_entry)); - memmove_s(begin + sizeof(struct fdt_reserve_entry), old_size, begin, - old_size); - e->address = htobe64(addr); - e->size = htobe64(len); -} - -size_t fdt_header_size(void) -{ - return sizeof(struct fdt_header); -} - -uint32_t fdt_total_size(struct fdt_header *hdr) -{ - return be32toh(hdr->totalsize); -} diff --git a/src/fdt_handler.c b/src/fdt_handler.c deleted file mode 100644 index db8c0ef6e..000000000 --- a/src/fdt_handler.c +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright 2018 The Hafnium Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hf/fdt_handler.h" - -#include "hf/boot_params.h" -#include "hf/cpu.h" -#include "hf/dlog.h" -#include "hf/fdt.h" -#include "hf/layout.h" -#include "hf/mm.h" -#include "hf/std.h" - -static uint64_t convert_number(const char *data, uint32_t size) -{ - union { - volatile uint64_t v; - char a[8]; - } t; - - switch (size) { - case sizeof(uint32_t): - return be32toh(*(uint32_t *)data); - case sizeof(uint64_t): - memcpy_s(t.a, sizeof(t.a), data, sizeof(uint64_t)); - return be64toh(t.v); - default: - return 0; - } -} - -static bool fdt_read_number(const struct fdt_node *node, const char *name, - uint64_t *value) -{ - const char *data; - uint32_t size; - - if (!fdt_read_property(node, name, &data, &size)) { - return false; - } - - switch (size) { - case sizeof(uint32_t): - case sizeof(uint64_t): - *value = convert_number(data, size); - break; - - default: - return false; - } - - return true; -} - -static bool fdt_write_number(struct fdt_node *node, const char *name, - uint64_t value) -{ - const char *data; - uint32_t size; - union { - volatile uint64_t v; - char a[8]; - } t; - - if (!fdt_read_property(node, name, &data, &size)) { - return false; - } - - switch (size) { - case sizeof(uint32_t): - *(uint32_t *)data = be32toh(value); - break; - - case sizeof(uint64_t): - t.v = be64toh(value); - memcpy_s((void *)data, size, t.a, sizeof(uint64_t)); - break; - - default: - return false; - } - - return true; -} - -/** - * Finds the memory region where initrd is stored, and updates the fdt node - * cursor to the node called "chosen". - */ -bool fdt_find_initrd(struct fdt_node *n, paddr_t *begin, paddr_t *end) -{ - uint64_t initrd_begin; - uint64_t initrd_end; - - if (!fdt_find_child(n, "chosen")) { - dlog("Unable to find 'chosen'\n"); - return false; - } - - if (!fdt_read_number(n, "linux,initrd-start", &initrd_begin)) { - dlog("Unable to read linux,initrd-start\n"); - return false; - } - - if (!fdt_read_number(n, "linux,initrd-end", &initrd_end)) { - dlog("Unable to read linux,initrd-end\n"); - return false; - } - - *begin = pa_init(initrd_begin); - *end = pa_init(initrd_end); - - return true; -} - -void fdt_find_cpus(const struct fdt_node *root, cpu_id_t *cpu_ids, - size_t *cpu_count) -{ - struct fdt_node n = *root; - const char *name; - uint64_t address_size; - - *cpu_count = 0; - - if (!fdt_find_child(&n, "cpus")) { - dlog("Unable to find 'cpus'\n"); - return; - } - - if (fdt_read_number(&n, "#address-cells", &address_size)) { - address_size *= sizeof(uint32_t); - } else { - address_size = sizeof(uint32_t); - } - - if (!fdt_first_child(&n, &name)) { - return; - } - - do { - const char *data; - uint32_t size; - - if (!fdt_read_property(&n, "device_type", &data, &size) || - size != sizeof("cpu") || - memcmp(data, "cpu", sizeof("cpu")) != 0 || - !fdt_read_property(&n, "reg", &data, &size)) { - continue; - } - - /* Get all entries for this CPU. */ - while (size >= address_size) { - if (*cpu_count >= MAX_CPUS) { - dlog("Found more than %d CPUs\n", MAX_CPUS); - return; - } - - cpu_ids[(*cpu_count)++] = - convert_number(data, address_size); - - size -= address_size; - data += address_size; - } - } while (fdt_next_sibling(&n, &name)); -} - -void fdt_find_memory_ranges(const struct fdt_node *root, struct boot_params *p) -{ - struct fdt_node n = *root; - const char *name; - uint64_t address_size; - uint64_t size_size; - uint64_t entry_size; - size_t mem_range_index = 0; - - /* Get the sizes of memory range addresses and sizes. */ - if (fdt_read_number(&n, "#address-cells", &address_size)) { - address_size *= sizeof(uint32_t); - } else { - address_size = sizeof(uint32_t); - } - - if (fdt_read_number(&n, "#size-cells", &size_size)) { - size_size *= sizeof(uint32_t); - } else { - size_size = sizeof(uint32_t); - } - - entry_size = address_size + size_size; - - /* Look for nodes with the device_type set to "memory". */ - if (!fdt_first_child(&n, &name)) { - return; - } - - do { - const char *data; - uint32_t size; - - if (!fdt_read_property(&n, "device_type", &data, &size) || - size != sizeof("memory") || - memcmp(data, "memory", sizeof("memory")) != 0 || - !fdt_read_property(&n, "reg", &data, &size)) { - continue; - } - - /* Traverse all memory ranges within this node. */ - while (size >= entry_size) { - uintpaddr_t addr = convert_number(data, address_size); - size_t len = - convert_number(data + address_size, size_size); - - if (mem_range_index < MAX_MEM_RANGES) { - p->mem_ranges[mem_range_index].begin = - pa_init(addr); - p->mem_ranges[mem_range_index].end = - pa_init(addr + len); - ++mem_range_index; - } else { - dlog("Found memory range %u in FDT but only " - "%u supported, ignoring additional range " - "of size %u.\n", - mem_range_index, MAX_MEM_RANGES, len); - } - - size -= entry_size; - data += entry_size; - } - } while (fdt_next_sibling(&n, &name)); - p->mem_ranges_count = mem_range_index; - - /* TODO: Check for "reserved-memory" nodes. */ -} - -struct fdt_header *fdt_map(struct mm_stage1_locked stage1_locked, - paddr_t fdt_addr, struct fdt_node *n, - struct mpool *ppool) -{ - struct fdt_header *fdt; - - /* Map the fdt header in. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_header_size()), MM_MODE_R, - ppool); - if (!fdt) { - dlog("Unable to map FDT header.\n"); - return NULL; - } - - if (!fdt_root_node(n, fdt)) { - dlog("FDT failed validation.\n"); - goto fail; - } - - /* Map the rest of the fdt in. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt)), MM_MODE_R, - ppool); - if (!fdt) { - dlog("Unable to map full FDT.\n"); - goto fail; - } - - return fdt; - -fail: - mm_unmap(stage1_locked, fdt_addr, pa_add(fdt_addr, fdt_header_size()), - ppool); - return NULL; -} - -bool fdt_unmap(struct mm_stage1_locked stage1_locked, struct fdt_header *fdt, - struct mpool *ppool) -{ - paddr_t fdt_addr = pa_from_va(va_from_ptr(fdt)); - - return mm_unmap(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt)), ppool); -} - -bool fdt_patch(struct mm_stage1_locked stage1_locked, paddr_t fdt_addr, - struct boot_params_update *p, struct mpool *ppool) -{ - struct fdt_header *fdt; - struct fdt_node n; - bool ret = false; - size_t i; - - /* Map the fdt header in. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_header_size()), MM_MODE_R, - ppool); - if (!fdt) { - dlog("Unable to map FDT header.\n"); - return false; - } - - if (!fdt_root_node(&n, fdt)) { - dlog("FDT failed validation.\n"); - goto err_unmap_fdt_header; - } - - /* Map the fdt (+ a page) in r/w mode in preparation for updating it. */ - fdt = mm_identity_map(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt) + PAGE_SIZE), - MM_MODE_R | MM_MODE_W, ppool); - if (!fdt) { - dlog("Unable to map FDT in r/w mode.\n"); - goto err_unmap_fdt_header; - } - - if (!fdt_find_child(&n, "")) { - dlog("Unable to find FDT root node.\n"); - goto out_unmap_fdt; - } - - if (!fdt_find_child(&n, "chosen")) { - dlog("Unable to find 'chosen'\n"); - goto out_unmap_fdt; - } - - /* Patch FDT to point to new ramdisk. */ - if (!fdt_write_number(&n, "linux,initrd-start", - pa_addr(p->initrd_begin))) { - dlog("Unable to write linux,initrd-start\n"); - goto out_unmap_fdt; - } - - if (!fdt_write_number(&n, "linux,initrd-end", pa_addr(p->initrd_end))) { - dlog("Unable to write linux,initrd-end\n"); - goto out_unmap_fdt; - } - - /* - * Patch FDT to reserve hypervisor memory so the primary VM doesn't try - * to use it. - */ - fdt_add_mem_reservation( - fdt, pa_addr(layout_text_begin()), - pa_difference(layout_text_begin(), layout_text_end())); - fdt_add_mem_reservation( - fdt, pa_addr(layout_rodata_begin()), - pa_difference(layout_rodata_begin(), layout_rodata_end())); - fdt_add_mem_reservation( - fdt, pa_addr(layout_data_begin()), - pa_difference(layout_data_begin(), layout_data_end())); - - /* Patch FDT to reserve memory for secondary VMs. */ - for (i = 0; i < p->reserved_ranges_count; ++i) { - fdt_add_mem_reservation( - fdt, pa_addr(p->reserved_ranges[i].begin), - pa_addr(p->reserved_ranges[i].end) - - pa_addr(p->reserved_ranges[i].begin)); - } - - ret = true; - -out_unmap_fdt: - /* Unmap FDT. */ - if (!mm_unmap(stage1_locked, fdt_addr, - pa_add(fdt_addr, fdt_total_size(fdt) + PAGE_SIZE), - ppool)) { - dlog("Unable to unmap writable FDT.\n"); - return false; - } - return ret; - -err_unmap_fdt_header: - mm_unmap(stage1_locked, fdt_addr, pa_add(fdt_addr, fdt_header_size()), - ppool); - return false; -} From 172c40acb9696368c546c66278eef0742e0f43d2 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Mon, 2 Sep 2019 00:00:38 +0900 Subject: [PATCH 07/45] Format and fix to call fdt_handler::{map, unmap, patch}. --- hfo2/src/fdt.rs | 21 ++++++++++++++------- hfo2/src/fdt_handler.rs | 34 +++++++++++++++++++++++++++------- hfo2/src/plat.rs | 11 +++++------ 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index f8c0951c2..105d7b9ae 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -429,11 +429,10 @@ pub unsafe extern "C" fn fdt_dump(hdr: *mut FdtHeader) { #[no_mangle] pub unsafe extern "C" fn fdt_root_node(node: *mut FdtNode, hdr: *const FdtHeader) -> bool { - FdtNode::new_root(&*hdr) - .map_or(false, |n| { - ptr::write(node, n); - true - }) + FdtNode::new_root(&*hdr).map_or(false, |n| { + ptr::write(node, n); + true + }) } #[no_mangle] @@ -450,7 +449,10 @@ pub unsafe extern "C" fn fdt_first_child(node: *mut FdtNode, child_name: *mut *c } #[no_mangle] -pub unsafe extern "C" fn fdt_next_sibling(node: *mut FdtNode, sibling_name: *mut *const u8) -> bool { +pub unsafe extern "C" fn fdt_next_sibling( + node: *mut FdtNode, + sibling_name: *mut *const u8, +) -> bool { (*node).next_sibling().map_or(false, |name| { ptr::write(sibling_name, name); true @@ -458,7 +460,12 @@ pub unsafe extern "C" fn fdt_next_sibling(node: *mut FdtNode, sibling_name: *mut } #[no_mangle] -pub unsafe extern "C" fn fdt_read_property(node: *mut FdtNode, name: *const u8, buf: *mut *const u8, size: *mut u32) -> bool { +pub unsafe extern "C" fn fdt_read_property( + node: *mut FdtNode, + name: *const u8, + buf: *mut *const u8, + size: *mut u32, +) -> bool { (*node).read_property(name, &mut *buf, &mut *size) } diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index 219a3cb67..1c9bb1040 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -425,8 +425,12 @@ pub unsafe fn patch( } #[no_mangle] -pub unsafe extern "C" fn fdt_map(stage1_locked: mm_stage1_locked, - fdt_addr: paddr_t, n: *mut FdtNode, ppool: *mut MPool) -> *mut FdtHeader { +pub unsafe extern "C" fn fdt_map( + mut stage1_locked: mm_stage1_locked, + fdt_addr: paddr_t, + n: *mut FdtNode, + ppool: *mut MPool, +) -> *mut FdtHeader { match map(&mut stage1_locked, fdt_addr, &mut *n, &mut *ppool) { Some(ret) => ret.as_ptr(), None => ptr::null_mut(), @@ -434,13 +438,20 @@ pub unsafe extern "C" fn fdt_map(stage1_locked: mm_stage1_locked, } #[no_mangle] -pub unsafe extern "C" fn fdt_unmap(stage1_locked: mm_stage1_locked, - fdt: *mut FdtHeader, ppool: *mut MPool) -> bool { +pub unsafe extern "C" fn fdt_unmap( + mut stage1_locked: mm_stage1_locked, + fdt: *mut FdtHeader, + ppool: *mut MPool, +) -> bool { unmap(&mut stage1_locked, &mut *fdt, &mut *ppool).is_ok() } #[no_mangle] -pub unsafe extern "C" fn fdt_find_cpus(root: *const FdtNode, cpu_ids: *mut cpu_id_t, cpu_count: *mut usize) { +pub unsafe extern "C" fn fdt_find_cpus( + root: *const FdtNode, + cpu_ids: *mut cpu_id_t, + cpu_count: *mut usize, +) { (*root).find_cpus(cpu_ids, &mut *cpu_count); } @@ -450,11 +461,20 @@ pub unsafe extern "C" fn fdt_find_memory_ranges(root: *const FdtNode, p: *mut Bo } #[no_mangle] -pub unsafe extern "C" fn fdt_find_initrd(n: *mut FdtNode, begin: *mut paddr_t, end: *mut paddr_t) -> bool { +pub unsafe extern "C" fn fdt_find_initrd( + n: *mut FdtNode, + begin: *mut paddr_t, + end: *mut paddr_t, +) -> bool { (*n).find_initrd(&mut *begin, &mut *end) } #[no_mangle] -pub unsafe extern "C" fn fdt_patch(stage1_locked: mm_stage1_locked, fdt_addr: paddr_t, p: *mut BootParamsUpdate, ppool: *mut MPool) -> bool { +pub unsafe extern "C" fn fdt_patch( + mut stage1_locked: mm_stage1_locked, + fdt_addr: paddr_t, + p: *mut BootParamsUpdate, + ppool: *mut MPool, +) -> bool { patch(&mut stage1_locked, fdt_addr, &mut *p, &mut *ppool).is_ok() } diff --git a/hfo2/src/plat.rs b/hfo2/src/plat.rs index a35269b8a..66f1993aa 100644 --- a/hfo2/src/plat.rs +++ b/hfo2/src/plat.rs @@ -22,8 +22,7 @@ use core::mem; use crate::addr::*; use crate::arch::*; use crate::boot_params::*; -use crate::fdt::*; -use crate::fdt_handler::*; +use crate::fdt_handler; use crate::layout::*; use crate::mm::*; use crate::mpool::*; @@ -67,7 +66,7 @@ pub unsafe extern "C" fn plat_get_boot_params( p.kernel_arg = plat_get_kernel_arg(); // Get the memory map from the FDT. - let fdt = fdt_map(ptable, plat_get_fdt_addr(), &mut n, ppool); + let fdt = fdt_handler::map(ptable, plat_get_fdt_addr(), &mut n, ppool); if fdt.is_none() { return false; } @@ -76,7 +75,7 @@ pub unsafe extern "C" fn plat_get_boot_params( if n.find_child("\0".as_ptr()).is_none() { dlog!("Unable to find FDT root node.\n"); // goto out_unmap_fdt; - if fdt_unmap(ptable, fdt, ppool).is_err() { + if fdt_handler::unmap(ptable, fdt, ppool).is_err() { dlog!("Unable to unmap fdt."); return false; } @@ -92,7 +91,7 @@ pub unsafe extern "C" fn plat_get_boot_params( ret = true; // out_unmap_fdt: - if fdt_unmap(ptable, fdt, ppool).is_err() { + if fdt_handler::unmap(ptable, fdt, ppool).is_err() { dlog!("Unable to unmap fdt."); return false; } @@ -113,5 +112,5 @@ pub unsafe fn plat_update_boot_params( p: &mut BootParamsUpdate, ppool: &mut MPool, ) -> bool { - fdt_patch(ptable, plat_get_fdt_addr(), p, ppool).is_ok() + fdt_handler::patch(ptable, plat_get_fdt_addr(), p, ppool).is_ok() } From 05accd4100719d2d2a38f6e390b3115f243a5401 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 4 Sep 2019 13:10:27 +0900 Subject: [PATCH 08/45] Remove plat.rs, since `#pragma weak` is not supported in Rust. Make safe wrapper functions instead. --- hfo2/src/boot_params.rs | 64 ++++++++++++++++++++++ hfo2/src/fdt_handler.rs | 6 +-- hfo2/src/init.rs | 21 ++++---- hfo2/src/lib.rs | 1 - hfo2/src/mm.rs | 6 +++ hfo2/src/plat.rs | 116 ---------------------------------------- hfo2/src/spinlock.rs | 6 +++ inc/hf/fdt_handler.h | 2 +- 8 files changed, 89 insertions(+), 133 deletions(-) delete mode 100644 hfo2/src/plat.rs diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs index aaa219784..b82763549 100644 --- a/hfo2/src/boot_params.rs +++ b/hfo2/src/boot_params.rs @@ -14,20 +14,35 @@ * limitations under the License. */ +use core::mem::MaybeUninit; + use crate::addr::*; use crate::arch::*; use crate::mm::*; use crate::mpool::*; +use crate::spinlock::*; use crate::types::*; pub const MAX_MEM_RANGES: usize = 20; #[derive(Clone)] +#[repr(C)] pub struct MemRange { pub begin: paddr_t, pub end: paddr_t, } +/// TODO(HfO2): Is it more natural for `paddr_t` to have 0 as default value? +impl Default for MemRange { + fn default() -> Self { + Self { + begin: pa_init(0), + end: pa_init(0), + } + } +} + +#[repr(C)] pub struct BootParams { pub cpu_ids: [cpu_id_t; MAX_CPUS], pub cpu_count: usize, @@ -38,9 +53,58 @@ pub struct BootParams { pub kernel_arg: uintreg_t, } +#[repr(C)] pub struct BootParamsUpdate { pub reserved_ranges: [MemRange; MAX_MEM_RANGES], pub reserved_ranges_count: usize, pub initrd_begin: paddr_t, pub initrd_end: paddr_t, } + +impl BootParamsUpdate { + pub fn new(initrd_begin: paddr_t, initrd_end: paddr_t) -> Self { + Self { + reserved_ranges: Default::default(), + reserved_ranges_count: 0, + initrd_begin, + initrd_end, + } + } +} + +extern "C" { + fn plat_get_boot_params( + stage1_locked: mm_stage1_locked, + p: *mut BootParams, + ppool: *mut MPool, + ) -> bool; + + fn plat_update_boot_params( + stage1_locked: mm_stage1_locked, + p: *mut BootParamsUpdate, + ppool: *mut MPool, + ) -> bool; +} + +pub fn get( + ptable: &mut SpinLockGuard>, + ppool: &mut MPool, +) -> Option { + unsafe { + let mut p: MaybeUninit = MaybeUninit::uninit(); + + if plat_get_boot_params(mm_stage1_locked::from_ref(ptable), p.get_mut(), ppool) { + Some(p.assume_init()) + } else { + None + } + } +} + +pub fn update( + ptable: &mut SpinLockGuard>, + p: &mut BootParamsUpdate, + ppool: &mut MPool, +) -> bool { + unsafe { plat_update_boot_params(mm_stage1_locked::from_ref(ptable), p, ppool) } +} diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index 1c9bb1040..352eb194e 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -299,7 +299,7 @@ pub unsafe fn unmap( pub unsafe fn patch( stage1_ptable: &mut PageTable, fdt_addr: paddr_t, - p: &mut BootParamsUpdate, + p: &BootParamsUpdate, ppool: &mut MPool, ) -> Result<(), ()> { // Map the fdt header in. @@ -473,8 +473,8 @@ pub unsafe extern "C" fn fdt_find_initrd( pub unsafe extern "C" fn fdt_patch( mut stage1_locked: mm_stage1_locked, fdt_addr: paddr_t, - p: *mut BootParamsUpdate, + p: *const BootParamsUpdate, ppool: *mut MPool, ) -> bool { - patch(&mut stage1_locked, fdt_addr, &mut *p, &mut *ppool).is_ok() + patch(&mut stage1_locked, fdt_addr, &*p, &mut *ppool).is_ok() } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 89135b585..31c0620ee 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -18,14 +18,13 @@ use core::mem::{self, MaybeUninit}; use crate::addr::*; use crate::api::*; -use crate::boot_params::*; +use crate::boot_params::{self, BootParamsUpdate}; use crate::cpu::*; use crate::load::*; use crate::memiter::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; -use crate::plat::*; use crate::types::*; use crate::vm::*; @@ -65,11 +64,9 @@ unsafe fn one_time_init() { mpool_enable_locks(); let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); - let mut params = mem::uninitialized(); - if !plat_get_boot_params(&mut hypervisor_ptable, &mut params, &mut ppool) { - panic!("unable to retrieve boot params"); - } + let mut params = boot_params::get(&mut hypervisor_ptable, &mut ppool) + .unwrap_or_else(|| panic!("unable to retrieve boot params")); cpu_module_init(&mut params.cpu_ids[0], params.cpu_count); @@ -117,13 +114,13 @@ unsafe fn one_time_init() { panic!("unable to load primary VM"); } - let mut update: BootParamsUpdate = mem::uninitialized(); - // load_secondary will add regions assigned to the secondary VMs from // mem_ranges to reserved_ranges. - update.initrd_begin = pa_from_va(va_from_ptr(primary_initrd.next as usize as *const _)); - update.initrd_end = pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)); - update.reserved_ranges_count = 0; + let mut update: BootParamsUpdate = BootParamsUpdate::new( + pa_from_va(va_from_ptr(primary_initrd.next as usize as *const _)), + pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)) + ); + if !load_secondary( &mut hypervisor_ptable, &cpio, @@ -135,7 +132,7 @@ unsafe fn one_time_init() { } // Prepare to run by updating bootparams as seen by primary VM. - if !plat_update_boot_params(&mut hypervisor_ptable, &mut update, &mut ppool) { + if !boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool) { panic!("plat_update_boot_params failed"); } diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index efa957fee..cd8369dfe 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -56,7 +56,6 @@ mod mm; mod mpool; mod page; mod panic; -mod plat; mod slist; mod spci; mod spci_architected_message; diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index cc0a3db19..15ec338b2 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -984,6 +984,12 @@ pub struct mm_stage1_locked { plock: usize, } +impl mm_stage1_locked { + pub fn from_ref(guard: &mut SpinLockGuard>) -> Self { + Self { plock: guard.raw() } + } +} + impl Deref for mm_stage1_locked { type Target = PageTable; diff --git a/hfo2/src/plat.rs b/hfo2/src/plat.rs deleted file mode 100644 index 66f1993aa..000000000 --- a/hfo2/src/plat.rs +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2019 Sanguk Park - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! TODO(HfO2): Functions here were denoted by `#pragma weak`, but no other -//! definitions are in Hafnium C code. How can I denote this? - -use core::mem; - -use crate::addr::*; -use crate::arch::*; -use crate::boot_params::*; -use crate::fdt_handler; -use crate::layout::*; -use crate::mm::*; -use crate::mpool::*; - -/// Default implementation assumes the FDT has been linked into the image. -/// -/// This can be overridden e.g. to provide a fixed address or an address passed -/// by the loader. -fn plat_get_fdt_addr() -> paddr_t { - unsafe { layout_fdt_begin() } -} - -/// Default implementation assumes the initrd has been linked into the image. -/// -/// This can be overridden e.g. to provide a fixed address or an address passed -/// by the loader. -fn plat_get_initrd_range(begin: &mut paddr_t, end: &mut paddr_t) { - *begin = unsafe { layout_initrd_begin() }; - *end = unsafe { layout_initrd_end() }; -} - -/// Default implementation assumes the FDT address is passed to the kernel. -/// -/// TODO: make this part of the VM configuration as secondary VMs will also need -/// to take arguments. -fn plat_get_kernel_arg() -> uintreg_t { - pa_addr(plat_get_fdt_addr()) as uintreg_t -} - -/// Default implementation extracts the boot parameters from the FDT but the -/// initrd is provided separately. -pub unsafe extern "C" fn plat_get_boot_params( - ptable: &mut PageTable, - p: &mut BootParams, - ppool: &mut MPool, -) -> bool { - let mut ret = false; - let mut n = mem::uninitialized(); - - plat_get_initrd_range(&mut p.initrd_begin, &mut p.initrd_end); - p.kernel_arg = plat_get_kernel_arg(); - - // Get the memory map from the FDT. - let fdt = fdt_handler::map(ptable, plat_get_fdt_addr(), &mut n, ppool); - if fdt.is_none() { - return false; - } - let fdt = fdt.unwrap().as_ptr(); - - if n.find_child("\0".as_ptr()).is_none() { - dlog!("Unable to find FDT root node.\n"); - // goto out_unmap_fdt; - if fdt_handler::unmap(ptable, fdt, ppool).is_err() { - dlog!("Unable to unmap fdt."); - return false; - } - - return ret; - } - - n.find_cpus(p.cpu_ids.as_mut_ptr(), &mut p.cpu_count); - - p.mem_ranges_count = 0; - n.find_memory_ranges(p); - - ret = true; - - // out_unmap_fdt: - if fdt_handler::unmap(ptable, fdt, ppool).is_err() { - dlog!("Unable to unmap fdt."); - return false; - } - - return ret; -} - -/// Default implementation updates the FDT which is the argument passed to the -/// primary VM's kernel. -/// -/// TODO: in future, each VM will declare whether it expects an argument passed -/// and that will be static data e.g. it will provide its own FDT so there will -/// be no FDT modification. This is done because each VM has a very different -/// view of the system and we don't want to force VMs to require loader code -/// when another loader can load the data for it. -pub unsafe fn plat_update_boot_params( - ptable: &mut PageTable, - p: &mut BootParamsUpdate, - ppool: &mut MPool, -) -> bool { - fdt_handler::patch(ptable, plat_get_fdt_addr(), p, ppool).is_ok() -} diff --git a/hfo2/src/spinlock.rs b/hfo2/src/spinlock.rs index 1bff66a7d..1a49f6f76 100644 --- a/hfo2/src/spinlock.rs +++ b/hfo2/src/spinlock.rs @@ -141,6 +141,12 @@ pub struct SpinLockGuard<'s, T> { unsafe impl<'s, T> Send for SpinLockGuard<'s, T> {} unsafe impl<'s, T: Send + Sync> Sync for SpinLockGuard<'s, T> {} +impl<'s, T> SpinLockGuard<'s, T> { + pub fn raw(&mut self) -> usize { + self.lock as *const _ as usize + } +} + impl<'s, T> Drop for SpinLockGuard<'s, T> { fn drop(&mut self) { self.lock.lock.unlock(); diff --git a/inc/hf/fdt_handler.h b/inc/hf/fdt_handler.h index 49d242e5b..8adee755b 100644 --- a/inc/hf/fdt_handler.h +++ b/inc/hf/fdt_handler.h @@ -33,4 +33,4 @@ bool fdt_find_initrd(struct fdt_node *n, paddr_t *begin, paddr_t *end); /** Apply an update to the FDT. */ bool fdt_patch(struct mm_stage1_locked stage1_locked, paddr_t fdt_addr, - struct boot_params_update *p, struct mpool *ppool); + const struct boot_params_update *p, struct mpool *ppool); From 77b1dc88d9f99a55177ddb882ddb718e9393cf77 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 4 Sep 2019 13:35:18 +0900 Subject: [PATCH 09/45] Refactor init, load, boot_params --- hfo2/src/boot_params.rs | 13 +++++++------ hfo2/src/init.rs | 23 ++++++++--------------- hfo2/src/load.rs | 38 ++++++++++++++++++++------------------ 3 files changed, 35 insertions(+), 39 deletions(-) diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs index b82763549..e3547be14 100644 --- a/hfo2/src/boot_params.rs +++ b/hfo2/src/boot_params.rs @@ -86,10 +86,7 @@ extern "C" { ) -> bool; } -pub fn get( - ptable: &mut SpinLockGuard>, - ppool: &mut MPool, -) -> Option { +pub fn get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> Option { unsafe { let mut p: MaybeUninit = MaybeUninit::uninit(); @@ -105,6 +102,10 @@ pub fn update( ptable: &mut SpinLockGuard>, p: &mut BootParamsUpdate, ppool: &mut MPool, -) -> bool { - unsafe { plat_update_boot_params(mm_stage1_locked::from_ref(ptable), p, ppool) } +) -> Result<(), ()> { + if unsafe { plat_update_boot_params(mm_stage1_locked::from_ref(ptable), p, ppool) } { + Ok(()) + } else { + Err(()) + } } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 31c0620ee..7155c1cc1 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -101,38 +101,31 @@ unsafe fn one_time_init() { pa_difference(params.initrd_begin, params.initrd_end), ); - let mut primary_initrd = mem::uninitialized(); - // Load all VMs. - if !load_primary( - &mut hypervisor_ptable, - &cpio, - params.kernel_arg, - &mut primary_initrd, - &mut ppool, - ) { - panic!("unable to load primary VM"); - } + let primary_initrd = load_primary(&mut hypervisor_ptable, &cpio, params.kernel_arg, &mut ppool) + .unwrap_or_else(|_| panic!("unable to load primary VM")); // load_secondary will add regions assigned to the secondary VMs from // mem_ranges to reserved_ranges. let mut update: BootParamsUpdate = BootParamsUpdate::new( pa_from_va(va_from_ptr(primary_initrd.next as usize as *const _)), - pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)) + pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)), ); - if !load_secondary( + if load_secondary( &mut hypervisor_ptable, &cpio, ¶ms, &mut update, &mut ppool, - ) { + ) + .is_err() + { panic!("unable to load secondary VMs"); } // Prepare to run by updating bootparams as seen by primary VM. - if !boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool) { + if boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool).is_err() { panic!("plat_update_boot_params failed"); } diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index e7d006b61..57c1c6801 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -14,7 +14,7 @@ * limitations under the License. */ -use core::mem; +use core::mem::{self, MaybeUninit}; use core::ptr; use core::slice; @@ -68,15 +68,14 @@ pub unsafe fn load_primary( hypervisor_ptable: &mut PageTable, cpio: *const MemIter, kernel_arg: uintreg_t, - initrd: *mut MemIter, ppool: &mut MPool, -) -> bool { +) -> Result { let mut it = mem::uninitialized(); let primary_begin = layout_primary_begin(); if !cpio_find_file(cpio, "vmlinuz\0".as_ptr(), &mut it) { dlog!("Unable to find vmlinuz\n"); - return false; + return Err(()); } dlog!( @@ -91,24 +90,27 @@ pub unsafe fn load_primary( ppool, ) { dlog!("Unable to relocate kernel for primary vm.\n"); - return false; + return Err(()); } - if !cpio_find_file(cpio, "initrd.img\0".as_ptr(), initrd) { + let mut initrd = MaybeUninit::uninit(); + if !cpio_find_file(cpio, "initrd.img\0".as_ptr(), initrd.get_mut()) { dlog!("Unable to find initrd.img\n"); - return false; + return Err(()); } + let initrd = initrd.assume_init(); + { let mut vm = mem::uninitialized(); if !vm_init(MAX_CPUS as u16, ppool, &mut vm) { dlog!("Unable to initialise primary vm\n"); - return false; + return Err(()); } if (*vm).id != HF_PRIMARY_VM_ID { dlog!("Primary vm was not given correct id\n"); - return false; + return Err(()); } // Map the 1TB of memory. @@ -122,12 +124,12 @@ pub unsafe fn load_primary( ppool, ) { dlog!("Unable to initialise memory for primary vm\n"); - return false; + return Err(()); } if !mm_vm_unmap_hypervisor(&mut (*vm).inner.get_mut_unchecked().ptable, ppool) { dlog!("Unable to unmap hypervisor from primary vm\n"); - return false; + return Err(()); } (*vm_get_vcpu(vm, 0)) @@ -136,7 +138,7 @@ pub unsafe fn load_primary( .on(ipa_from_pa(primary_begin), kernel_arg); } - true + Ok(initrd) } /// Try to find a memory range of the given size within the given ranges, and @@ -177,7 +179,7 @@ unsafe fn update_reserved_ranges( before: *const MemRange, after: *const MemRange, mem_ranges_count: usize, -) -> bool { +) -> Result<(), ()> { let after = slice::from_raw_parts(after, mem_ranges_count); let before = slice::from_raw_parts(before, mem_ranges_count); @@ -185,7 +187,7 @@ unsafe fn update_reserved_ranges( if pa_addr(after.begin) > pa_addr(before.begin) { if (*update).reserved_ranges_count >= MAX_MEM_RANGES { dlog!("Too many reserved ranges after loading secondary VMs.\n"); - return false; + return Err(()); } (*update).reserved_ranges[(*update).reserved_ranges_count].begin = after.end; @@ -194,7 +196,7 @@ unsafe fn update_reserved_ranges( } } - true + Ok(()) } /// Loads all secondary VMs into the memory ranges from the given params. @@ -205,7 +207,7 @@ pub unsafe fn load_secondary( params: *const BootParams, update: *mut BootParamsUpdate, ppool: &mut MPool, -) -> bool { +) -> Result<(), ()> { let mut mem_ranges_available: [MemRange; MAX_MEM_RANGES] = mem::uninitialized(); // static_assert( // sizeof(mem_ranges_available) == sizeof(params->mem_ranges), @@ -219,7 +221,7 @@ pub unsafe fn load_secondary( if !cpio_find_file(cpio, "vms.txt\0".as_ptr(), &mut it) { dlog!("vms.txt is missing\n"); - return true; + return Ok(()); } // Round the last addresses down to the page size. @@ -313,7 +315,7 @@ pub unsafe fn load_secondary( ppool, ) { dlog!("Unable to unmap secondary VM from primary VM\n"); - return false; + return Err(()); } dlog!( From d21425f536985524128d448a2fd79158afbd4ff0 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 4 Sep 2019 16:51:03 +0900 Subject: [PATCH 10/45] Fix wrongly ported code (!*p should be *p == 0) --- hfo2/src/fdt.rs | 34 ++++++++++++++++++---------------- hfo2/src/fdt_handler.rs | 20 ++++++++------------ hfo2/src/load.rs | 12 ++++++------ 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 105d7b9ae..3f6c0474b 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -86,7 +86,7 @@ impl FdtTokenizer { } unsafe fn u32(&mut self) -> Option { - let next = self.cur.offset(mem::size_of::() as isize); + let next = self.cur.add(mem::size_of::()); if next > self.end { return None; } @@ -107,7 +107,7 @@ impl FdtTokenizer { } unsafe fn bytes(&mut self, size: usize) -> Option<*const u8> { - let next = self.cur.offset(size as isize); + let next = self.cur.add(size); if next > self.end { return None; @@ -124,13 +124,15 @@ impl FdtTokenizer { let mut p = self.cur; while p < self.end { - if *p != 0 { + if *p == 0 { // Found the end of the string. let res = self.cur; - self.cur = p.offset(1); + self.cur = p.add(1); self.align(); return Some(res); } + + p = p.add(1); } None @@ -146,7 +148,7 @@ impl FdtTokenizer { if token != FdtToken::Prop as u32 { // Rewind so that caller will get the same token. - self.cur = self.cur.offset(-(mem::size_of::() as isize)); + self.cur = self.cur.sub(mem::size_of::()); return None; } @@ -160,7 +162,7 @@ impl FdtTokenizer { let buf = this.bytes(*size as usize)?; //TODO: Need to verify the strings. - *name = this.strs.offset(nameoff as isize); + *name = this.strs.add(nameoff as usize); mem::forget(this); Some(()) @@ -170,7 +172,7 @@ impl FdtTokenizer { let token = self.token()?; if token != FdtToken::BeginNode as u32 { // Rewind so that caller will get the same token. - self.cur = self.cur.offset(-(mem::size_of::() as isize)); + self.cur = self.cur.sub(mem::size_of::()); return None; } @@ -238,9 +240,9 @@ impl FdtNode { // TODO: Verify strings as well. Some(Self { hdr: ptr::null(), - begin: hdr_ptr.offset(begin as isize), - end: hdr_ptr.offset((begin + size) as isize), - strs: hdr_ptr.offset(u32::from_be(hdr.off_dt_strings) as isize), + begin: hdr_ptr.add(begin as usize), + end: hdr_ptr.add((begin + size) as usize), + strs: hdr_ptr.add(u32::from_be(hdr.off_dt_strings) as usize), }) } @@ -303,7 +305,7 @@ impl FdtHeader { pub unsafe fn dump(&self) { unsafe fn asciz_to_utf8(ptr: *const u8) -> &'static str { let mut len = 0; - while *ptr.offset(len as isize) != 0u8 { + while *ptr.add(len) != 0u8 { len += 1; } let bytes = slice::from_raw_parts(ptr, len); @@ -341,11 +343,11 @@ impl FdtHeader { 2 * depth, asciz_to_utf8(name) ); - for i in 0..size { + for i in 0..size as usize { dlog!( "{}{:02x}", if i == 0 { "" } else { " " }, - *buf.offset(i as isize) + *buf.add(i) ); } dlog!(")\n"); @@ -378,7 +380,7 @@ impl FdtHeader { u64::from_be((*entry).address) as *const u8, u64::from_be((*entry).size) ); - entry = entry.offset(1); + entry = entry.add(1); } } } @@ -386,7 +388,7 @@ impl FdtHeader { pub unsafe fn add_mem_reservation(&mut self, addr: u64, len: u64) { // TODO: Clean this up. let begin = (self as *const _ as usize as *const u8) - .offset(u32::from_be(self.off_mem_rsvmap) as isize); + .add(u32::from_be(self.off_mem_rsvmap) as usize); let e = begin as *mut FdtReserveEntry; let old_size = u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap); @@ -398,7 +400,7 @@ impl FdtHeader { (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); memmove_s( - begin.offset(mem::size_of::() as isize) as usize as *mut _, + begin.add(mem::size_of::()) as usize as *mut _, old_size as usize, begin as usize as *const _, old_size as usize, diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index 352eb194e..bb0d02d78 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -118,7 +118,7 @@ impl FdtNode { .map(|size| size as usize * mem::size_of::()) .unwrap_or(mem::size_of::()); - let mut name = node.first_child()?; + node.first_child()?; loop { let mut data = mem::uninitialized(); @@ -143,17 +143,15 @@ impl FdtNode { return None; } - *cpu_ids.offset(*cpu_count as isize) = + *cpu_ids.add(*cpu_count) = convert_number(data, address_size as u32) as cpu_id_t; *cpu_count += 1; size -= address_size as u32; - data = data.offset(address_size as isize); + data = data.add(address_size); } - if let Some(sibling_name) = node.next_sibling() { - name = sibling_name; - } else { + if node.next_sibling().is_none() { break; } } @@ -178,7 +176,7 @@ impl FdtNode { let entry_size = address_size + size_size; // Look for nodes with the device_type set to "memory". - let mut name = node.first_child()?; + node.first_child()?; let mut mem_range_index = 0; loop { @@ -201,7 +199,7 @@ impl FdtNode { while size as usize >= entry_size { let addr = convert_number(data, address_size as u32) as usize; let len = - convert_number(data.offset(address_size as isize), size_size as u32) as usize; + convert_number(data.add(address_size), size_size as u32) as usize; if mem_range_index < MAX_MEM_RANGES { p.mem_ranges[mem_range_index].begin = pa_init(addr); @@ -213,12 +211,10 @@ impl FdtNode { } size -= entry_size as u32; - data = data.offset(entry_size as isize); + data = data.add(entry_size); } - if let Some(sibling_name) = node.next_sibling() { - name = sibling_name; - } else { + if node.next_sibling().is_none() { break; } } diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 57c1c6801..baaf586da 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -153,15 +153,15 @@ unsafe fn carve_out_mem_range( ) -> bool { // TODO(b/116191358): Consider being cleverer about how we pack VMs // together, with a non-greedy algorithm. - for i in 0..mem_ranges_count as isize { + for i in 0..mem_ranges_count { if size_to_find - <= pa_difference((*mem_ranges.offset(i)).begin, (*mem_ranges.offset(i)).end) as u64 + <= pa_difference((*mem_ranges.add(i)).begin, (*mem_ranges.add(i)).end) as u64 { // This range is big enough, take some of it from the end and reduce // its size accordingly. - *found_end = (*mem_ranges.offset(i)).end; - *found_begin = pa_init(pa_addr((*mem_ranges.offset(i)).end) - size_to_find as usize); - (*mem_ranges.offset(i)).end = *found_begin; + *found_end = (*mem_ranges.add(i)).end; + *found_begin = pa_init(pa_addr((*mem_ranges.add(i)).end) - size_to_find as usize); + (*mem_ranges.add(i)).end = *found_begin; return true; } } @@ -242,7 +242,7 @@ pub unsafe fn load_secondary( let mut p = name.next; while p != name.limit { dlog!("{}", *p as char); - p = p.offset(1); + p = p.add(1); } dlog!("\n"); From c1e44861ebdf09d68bed42f68f5b726bef606911 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 4 Sep 2019 18:31:44 +0900 Subject: [PATCH 11/45] Fix wrongly ported code (write to the given reference, and continue evaluates condition expression in do-while loop) --- hfo2/src/fdt.rs | 2 +- hfo2/src/fdt_handler.rs | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 3f6c0474b..c4016c2d8 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -159,7 +159,7 @@ impl FdtTokenizer { *size = this.u32()?; let nameoff = this.u32()?; - let buf = this.bytes(*size as usize)?; + *buf = this.bytes(*size as usize)?; //TODO: Need to verify the strings. *name = this.strs.add(nameoff as usize); diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index bb0d02d78..e1cd53606 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -120,6 +120,7 @@ impl FdtNode { node.first_child()?; + // TODO(HfO2): this loop was do-while in C. Make an interator for this. loop { let mut data = mem::uninitialized(); let mut size = mem::uninitialized(); @@ -131,9 +132,13 @@ impl FdtNode { "cpu\0".as_ptr() as usize as *const _, "cpu\0".len(), ) != 0 - || !node.read_property("reg".as_ptr(), &mut data, &mut size) + || !node.read_property("reg\0".as_ptr(), &mut data, &mut size) { - continue; + if node.next_sibling().is_none() { + break; + } else { + continue; + } } // Get all entries for this CPU. @@ -179,6 +184,7 @@ impl FdtNode { node.first_child()?; let mut mem_range_index = 0; + // TODO(HfO2): this loop was do-while in C. Make an interator for this. loop { let mut data = mem::uninitialized(); let mut size = mem::uninitialized(); @@ -192,7 +198,11 @@ impl FdtNode { ) != 0 || !node.read_property("reg\0".as_ptr(), &mut data, &mut size) { - continue; + if node.next_sibling().is_none() { + break; + } else { + continue; + } } // Traverse all memory ranges within this node. From f2fba0f9683c0ec0d9e252c4170f087467a56693 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 4 Sep 2019 18:39:37 +0900 Subject: [PATCH 12/45] Use different constants for unit tests. --- hfo2/src/types.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hfo2/src/types.rs b/hfo2/src/types.rs index 41ae4a2b2..2acb08bc1 100644 --- a/hfo2/src/types.rs +++ b/hfo2/src/types.rs @@ -57,13 +57,20 @@ pub const HF_INVALID_INTID: intid_t = 0xffff_ffff; /// The virtual interrupt ID used for the virtual timer. pub const HF_VIRTUAL_TIMER_INTID: intid_t = 3; -// These constants are originally from build scripts. Fortunately most -// testing environments have same conditions (HEAP_PAGES=60, MAX_CPUS=8, -// MAX_VMS=16.) And only one environment (host_fake) doesn't but it's for the -// unit test, so works fine under this settting (See +// These constants are originally from build scripts. (See // //project/reference/BUILD.gn.) pub const HEAP_PAGES: usize = 60; + +#[cfg(target_arch = "x86_64")] +pub const MAX_CPUS: usize = 4; + +#[cfg(target_arch = "aarch64")] pub const MAX_CPUS: usize = 8; + +#[cfg(target_arch = "x86_64")] +pub const MAX_VMS: usize = 6; + +#[cfg(target_arch = "aarch64")] pub const MAX_VMS: usize = 16; /// The ID of the primary VM which is responsible for scheduling. From aa8020dd75608245e8c4ceba65632c1b79c57e08 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 4 Sep 2019 21:48:51 +0900 Subject: [PATCH 13/45] Resolve warnings on FDT and FDT handler. --- hfo2/src/fdt.rs | 84 +++++++++++++++++++++---------------- hfo2/src/fdt_handler.rs | 93 ++++++++++++++++++++++------------------- 2 files changed, 97 insertions(+), 80 deletions(-) diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index c4016c2d8..48c429471 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -18,6 +18,7 @@ use core::mem; use core::ptr; use core::slice; use core::str; +use core::convert::{TryFrom, TryInto}; use crate::std::*; use crate::utils::*; @@ -55,6 +56,7 @@ struct FdtReserveEntry { size: u64, } +#[derive(PartialEq)] enum FdtToken { BeginNode = 1, EndNode = 2, @@ -63,6 +65,23 @@ enum FdtToken { End = 9, } +impl TryFrom for FdtToken { + type Error = (); + + fn try_from(value: u32) -> Result { + use FdtToken::*; + + match value { + 1 => Ok(BeginNode), + 2 => Ok(EndNode), + 3 => Ok(Prop), + 4 => Ok(Nop), + 9 => Ok(End), + _ => Err(()), + } + } +} + struct FdtTokenizer { cur: *const u8, end: *const u8, @@ -96,10 +115,11 @@ impl FdtTokenizer { Some(res) } - unsafe fn token(&mut self) -> Option { + unsafe fn token(&mut self) -> Option { while let Some(v) = self.u32() { - if v != FdtToken::Nop as u32 { - return Some(v); + let token = v.try_into().unwrap(); + if token != FdtToken::Nop { + return Some(token); } } @@ -138,15 +158,10 @@ impl FdtTokenizer { None } - unsafe fn next_property( - &mut self, - name: &mut *const u8, - buf: &mut *const u8, - size: &mut u32, - ) -> Option<()> { + unsafe fn next_property(&mut self) -> Option<(*const u8, *const u8, u32)> { let token = self.token()?; - if token != FdtToken::Prop as u32 { + if token != FdtToken::Prop { // Rewind so that caller will get the same token. self.cur = self.cur.sub(mem::size_of::()); return None; @@ -157,20 +172,20 @@ impl FdtTokenizer { this.cur = this.end; }); - *size = this.u32()?; + let size = this.u32()?; let nameoff = this.u32()?; - *buf = this.bytes(*size as usize)?; + let buf = this.bytes(size as usize)?; //TODO: Need to verify the strings. - *name = this.strs.add(nameoff as usize); + let name = this.strs.add(nameoff as usize); mem::forget(this); - Some(()) + Some((name, buf, size)) } unsafe fn next_subnode(&mut self) -> Option<*const u8> { let token = self.token()?; - if token != FdtToken::BeginNode as u32 { + if token != FdtToken::BeginNode { // Rewind so that caller will get the same token. self.cur = self.cur.sub(mem::size_of::()); return None; @@ -188,10 +203,7 @@ impl FdtTokenizer { } unsafe fn skip_properties(&mut self) { - let mut name = mem::uninitialized(); - let mut buf = mem::uninitialized(); - let mut size = mem::uninitialized(); - while let Some(_) = self.next_property(&mut name, &mut buf, &mut size) {} + while self.next_property().is_some() {} } unsafe fn skip_node(&mut self) -> Option<()> { @@ -205,7 +217,7 @@ impl FdtTokenizer { } let token = self.token()?; - if token != FdtToken::EndNode as u32 { + if token != FdtToken::EndNode { self.cur = self.end; return None; } @@ -249,18 +261,15 @@ impl FdtNode { pub unsafe fn read_property( &self, name: *const u8, - buf: &mut *const u8, - size: &mut u32, - ) -> bool { - let mut prop_name = ptr::null(); + ) -> Result<(*const u8, u32), ()> { let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); - while let Some(_) = t.next_property(&mut prop_name, buf, size) { + while let Some((prop_name, buf, size)) = t.next_property() { if strcmp(prop_name, name) == 0 { - return true; + return Ok((buf, size)); } } - false + Err(()) } pub unsafe fn first_child(&mut self) -> Option<*const u8> { @@ -333,10 +342,7 @@ impl FdtHeader { asciz_to_utf8(name) ); depth += 1; - let mut name = mem::uninitialized(); - let mut buf = mem::uninitialized(); - let mut size = mem::uninitialized(); - while let Some(_) = t.next_property(&mut name, &mut buf, &mut size) { + while let Some((name, buf, size)) = t.next_property() { dlog!( "{:1$}property: \"{2}\" (", "", @@ -354,10 +360,9 @@ impl FdtHeader { } } - let token = match t.token().filter(|t| *t != FdtToken::EndNode as u32) { - Some(token) => token, - None => return, - }; + if t.token().filter(|t| *t != FdtToken::EndNode).is_none() { + return; + } depth -= 1; @@ -468,7 +473,14 @@ pub unsafe extern "C" fn fdt_read_property( buf: *mut *const u8, size: *mut u32, ) -> bool { - (*node).read_property(name, &mut *buf, &mut *size) + match (*node).read_property(name) { + Ok((prop_buf, prop_size)) => { + *buf = prop_buf; + *size = prop_size; + true + } + Err(_) => false, + } } #[no_mangle] diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index e1cd53606..98734ae51 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -39,27 +39,18 @@ unsafe fn convert_number(data: *const u8, size: u32) -> u64 { } impl FdtNode { - unsafe fn read_number(&mut self, name: *const u8) -> Option { - let mut data = mem::uninitialized(); - let mut size = mem::uninitialized(); - - if !self.read_property(name, &mut data, &mut size) { - return None; - } + unsafe fn read_number(&mut self, name: *const u8) -> Result { + let (data, size) = self.read_property(name)?; match size { - 4 | 8 => Some(convert_number(data, size)), - _ => None, + 4 | 8 => Ok(convert_number(data, size)), + _ => Err(()), } } - unsafe fn write_number(&mut self, name: *const u8, value: u64) -> bool { - let mut data = mem::uninitialized(); - let mut size = mem::uninitialized(); + unsafe fn write_number(&mut self, name: *const u8, value: u64) -> Result<(), ()> { - if !self.read_property(name, &mut data, &mut size) { - return false; - } + let (data, size) = self.read_property(name)?; match size { 4 => { @@ -68,10 +59,10 @@ impl FdtNode { 8 => { *(data as *mut u64) = u64::from_be(value); } - _ => return false, + _ => return Err(()), } - true + Ok(()) } /// Finds the memory region where initrd is stored, and updates the fdt node @@ -83,16 +74,16 @@ impl FdtNode { } let initrd_begin = match self.read_number("linux,initrd-start\0".as_ptr()) { - Some(initrd_begin) => initrd_begin, - None => { + Ok(initrd_begin) => initrd_begin, + Err(_) => { dlog!("Unable to read linux,initrd-start\n"); return false; } }; let initrd_end = match self.read_number("linux,initrd-end\0".as_ptr()) { - Some(initrd_end) => initrd_end, - None => { + Ok(initrd_end) => initrd_end, + Err(_) => { dlog!("Unable to read linux,initrd-end\n"); return false; } @@ -122,18 +113,15 @@ impl FdtNode { // TODO(HfO2): this loop was do-while in C. Make an interator for this. loop { - let mut data = mem::uninitialized(); - let mut size = mem::uninitialized(); - - if !node.read_property("device_type\0".as_ptr(), &mut data, &mut size) - || size != "cpu\0".len() as u32 - || memcmp_rs( - data as usize as *const _, + if node + .read_property("device_type\0".as_ptr()) + .ok() + .filter(|(_, size)| *size == "cpu\0".len() as u32) + .filter(|(data, _)| memcmp_rs( + *data as usize as *const _, "cpu\0".as_ptr() as usize as *const _, "cpu\0".len(), - ) != 0 - || !node.read_property("reg\0".as_ptr(), &mut data, &mut size) - { + ) == 0).is_none() { if node.next_sibling().is_none() { break; } else { @@ -141,6 +129,17 @@ impl FdtNode { } } + let (mut data, mut size) = if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) + { + (data, size) + } else { + if node.next_sibling().is_none() { + break; + } else { + continue; + } + }; + // Get all entries for this CPU. while size as usize >= address_size { if *cpu_count >= MAX_CPUS { @@ -186,24 +185,30 @@ impl FdtNode { // TODO(HfO2): this loop was do-while in C. Make an interator for this. loop { - let mut data = mem::uninitialized(); - let mut size = mem::uninitialized(); - - if !node.read_property("device_type\0".as_ptr(), &mut data, &mut size) - || size as usize != "memory\0".len() - || memcmp_rs( - data as usize as *const _, + if node + .read_property("device_type\0".as_ptr()) + .ok() + .filter(|(_, size)| *size as usize == "memory\0".len()) + .filter(|(data, _)| memcmp_rs( + *data as usize as *const _, "memory\0".as_ptr() as usize as *const _, "memory\0".len(), - ) != 0 - || !node.read_property("reg\0".as_ptr(), &mut data, &mut size) - { + ) == 0).is_none() { if node.next_sibling().is_none() { break; } else { continue; } } + let (mut data, mut size) = if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) { + (data, size) + } else { + if node.next_sibling().is_none() { + break; + } else { + continue; + } + }; // Traverse all memory ranges within this node. while size as usize >= entry_size { @@ -378,15 +383,15 @@ pub unsafe fn patch( } // Patch FDT to point to new ramdisk. - if !node.write_number( + if node.write_number( "linux,initrd-start\0".as_ptr(), pa_addr(p.initrd_begin) as u64, - ) { + ).is_err() { dlog!("Unable to write linux,initrd-start\n"); return Err(()); } - if !node.write_number("linux,initrd-end\0".as_ptr(), pa_addr(p.initrd_end) as u64) { + if node.write_number("linux,initrd-end\0".as_ptr(), pa_addr(p.initrd_end) as u64).is_err() { dlog!("Unable to write linux,initrd-end\n"); return Err(()); } From 4d1e6a90f0161c3582eceb48d9baf86ee53abdde Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 13:08:34 +0900 Subject: [PATCH 14/45] Make a singleton object representing MemoryManager. --- hfo2/src/api.rs | 3 +- hfo2/src/fdt.rs | 13 +--- hfo2/src/fdt_handler.rs | 86 +++++++++++++---------- hfo2/src/init.rs | 8 +-- hfo2/src/lib.rs | 1 + hfo2/src/mm.rs | 146 ++++++++++++++++++++++++---------------- hfo2/src/singleton.rs | 38 +++++++++++ hfo2/src/vm.rs | 5 +- 8 files changed, 188 insertions(+), 112 deletions(-) create mode 100644 hfo2/src/singleton.rs diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index da25bc2d6..0894563fa 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -34,6 +34,7 @@ use crate::std::*; use crate::types::*; use crate::utils::*; use crate::vm::*; +use crate::singleton::*; // To eliminate the risk of deadlocks, we define a partial order for the acquisition of locks held // concurrently by the same physical CPU. Our current ordering requirements are as follows: @@ -868,7 +869,7 @@ pub unsafe extern "C" fn api_interrupt_inject( /// Clears a region of physical memory by overwriting it with zeros. The data is /// flushed from the cache so the memory has been cleared across the system. fn clear_memory(begin: paddr_t, end: paddr_t, ppool: &MPool) -> Result<(), ()> { - let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); + let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.HYPERVISOR_PAGE_TABLE.lock(); let size = pa_difference(begin, end); let region = pa_addr(begin); diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 48c429471..7584d6e30 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -14,11 +14,11 @@ * limitations under the License. */ +use core::convert::{TryFrom, TryInto}; use core::mem; use core::ptr; use core::slice; use core::str; -use core::convert::{TryFrom, TryInto}; use crate::std::*; use crate::utils::*; @@ -258,10 +258,7 @@ impl FdtNode { }) } - pub unsafe fn read_property( - &self, - name: *const u8, - ) -> Result<(*const u8, u32), ()> { + pub unsafe fn read_property(&self, name: *const u8) -> Result<(*const u8, u32), ()> { let mut t = FdtTokenizer::new(self.strs, self.begin, self.end); while let Some((prop_name, buf, size)) = t.next_property() { if strcmp(prop_name, name) == 0 { @@ -350,11 +347,7 @@ impl FdtHeader { asciz_to_utf8(name) ); for i in 0..size as usize { - dlog!( - "{}{:02x}", - if i == 0 { "" } else { " " }, - *buf.add(i) - ); + dlog!("{}{:02x}", if i == 0 { "" } else { " " }, *buf.add(i)); } dlog!(")\n"); } diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index 98734ae51..886b8c5f5 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -49,7 +49,6 @@ impl FdtNode { } unsafe fn write_number(&mut self, name: *const u8, value: u64) -> Result<(), ()> { - let (data, size) = self.read_property(name)?; match size { @@ -117,11 +116,15 @@ impl FdtNode { .read_property("device_type\0".as_ptr()) .ok() .filter(|(_, size)| *size == "cpu\0".len() as u32) - .filter(|(data, _)| memcmp_rs( - *data as usize as *const _, - "cpu\0".as_ptr() as usize as *const _, - "cpu\0".len(), - ) == 0).is_none() { + .filter(|(data, _)| { + memcmp_rs( + *data as usize as *const _, + "cpu\0".as_ptr() as usize as *const _, + "cpu\0".len(), + ) == 0 + }) + .is_none() + { if node.next_sibling().is_none() { break; } else { @@ -129,16 +132,16 @@ impl FdtNode { } } - let (mut data, mut size) = if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) - { - (data, size) - } else { - if node.next_sibling().is_none() { - break; + let (mut data, mut size) = + if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) { + (data, size) } else { - continue; - } - }; + if node.next_sibling().is_none() { + break; + } else { + continue; + } + }; // Get all entries for this CPU. while size as usize >= address_size { @@ -147,8 +150,7 @@ impl FdtNode { return None; } - *cpu_ids.add(*cpu_count) = - convert_number(data, address_size as u32) as cpu_id_t; + *cpu_ids.add(*cpu_count) = convert_number(data, address_size as u32) as cpu_id_t; *cpu_count += 1; size -= address_size as u32; @@ -189,32 +191,36 @@ impl FdtNode { .read_property("device_type\0".as_ptr()) .ok() .filter(|(_, size)| *size as usize == "memory\0".len()) - .filter(|(data, _)| memcmp_rs( - *data as usize as *const _, - "memory\0".as_ptr() as usize as *const _, - "memory\0".len(), - ) == 0).is_none() { + .filter(|(data, _)| { + memcmp_rs( + *data as usize as *const _, + "memory\0".as_ptr() as usize as *const _, + "memory\0".len(), + ) == 0 + }) + .is_none() + { if node.next_sibling().is_none() { break; } else { continue; } } - let (mut data, mut size) = if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) { - (data, size) - } else { - if node.next_sibling().is_none() { - break; + let (mut data, mut size) = + if let Ok((data, size)) = node.read_property("reg\0".as_ptr()) { + (data, size) } else { - continue; - } - }; + if node.next_sibling().is_none() { + break; + } else { + continue; + } + }; // Traverse all memory ranges within this node. while size as usize >= entry_size { let addr = convert_number(data, address_size as u32) as usize; - let len = - convert_number(data.add(address_size), size_size as u32) as usize; + let len = convert_number(data.add(address_size), size_size as u32) as usize; if mem_range_index < MAX_MEM_RANGES { p.mem_ranges[mem_range_index].begin = pa_init(addr); @@ -383,15 +389,21 @@ pub unsafe fn patch( } // Patch FDT to point to new ramdisk. - if node.write_number( - "linux,initrd-start\0".as_ptr(), - pa_addr(p.initrd_begin) as u64, - ).is_err() { + if node + .write_number( + "linux,initrd-start\0".as_ptr(), + pa_addr(p.initrd_begin) as u64, + ) + .is_err() + { dlog!("Unable to write linux,initrd-start\n"); return Err(()); } - if node.write_number("linux,initrd-end\0".as_ptr(), pa_addr(p.initrd_end) as u64).is_err() { + if node + .write_number("linux,initrd-end\0".as_ptr(), pa_addr(p.initrd_end) as u64) + .is_err() + { dlog!("Unable to write linux,initrd-end\n"); return Err(()); } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 7155c1cc1..ace6bc409 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -25,6 +25,7 @@ use crate::memiter::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; +use crate::singleton::*; use crate::types::*; use crate::vm::*; @@ -55,15 +56,14 @@ unsafe fn one_time_init() { mem::size_of_val(PTABLE_BUF.get_ref()), ); - if !mm_init(&mut ppool) { - panic!("mm_init failed"); - } + let memory_manager = MemoryManager::new(&ppool).expect("mm_init failed"); + MEMORY_MANAGER = MaybeUninit::new(memory_manager); // Enable locks now that mm is initialised. dlog_enable_lock(); mpool_enable_locks(); - let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); + let mut hypervisor_ptable = MEMORY_MANAGER.get_ref().HYPERVISOR_PAGE_TABLE.lock(); let mut params = boot_params::get(&mut hypervisor_ptable, &mut ppool) .unwrap_or_else(|| panic!("unable to retrieve boot params")); diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index cd8369dfe..61b1e8dcd 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -56,6 +56,7 @@ mod mm; mod mpool; mod page; mod panic; +mod singleton; mod slist; mod spci; mod spci_architected_message; diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index 15ec338b2..c5e694e89 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -27,7 +27,7 @@ use core::cmp; use core::marker::PhantomData; -use core::mem; +use core::mem::{self, MaybeUninit}; use core::ops::*; use core::ptr; use core::slice; @@ -42,6 +42,7 @@ use crate::page::*; use crate::spinlock::{SpinLock, SpinLockGuard}; use crate::types::*; use crate::utils::*; +use crate::singleton::*; extern "C" { fn arch_mm_absent_pte(level: u8) -> pte_t; @@ -162,13 +163,6 @@ type ptable_addr_t = uintvaddr_t; // For stage 2, the input is an intermediate physical addresses rather than a virtual address so: const_assert_eq!(addr_size_eq; mem::size_of::(), mem::size_of::()); -/// The hypervisor page table. -pub static HYPERVISOR_PAGE_TABLE: SpinLock> = - SpinLock::new(unsafe { PageTable::null() }); - -/// Is stage2 invalidation enabled? -pub static STAGE2_INVALIDATE: AtomicBool = AtomicBool::new(false); - /// Utility functions for address manipulation. mod addr { use super::ptable_addr_t; @@ -270,7 +264,7 @@ impl Stage for Stage2 { } fn invalidate_tlb(begin: usize, end: usize) { - if STAGE2_INVALIDATE.load(Ordering::Relaxed) { + if unsafe { MEMORY_MANAGER.get_ref() }.STAGE2_INVALIDATE.load(Ordering::Relaxed) { unsafe { arch_mm_invalidate_stage2_range(ipa_init(begin), ipa_init(end)); } @@ -1024,6 +1018,80 @@ impl<'s> Into>> for mm_stage1_locked { } } +pub struct MemoryManager { + /// The hypervisor page table. + pub HYPERVISOR_PAGE_TABLE: SpinLock>, + + /// Is stage2 invalidation enabled? + pub STAGE2_INVALIDATE: AtomicBool, +} + +impl MemoryManager { + pub fn new(mpool: &MPool) -> Option { + dlog!( + "text: {:#x} - {:#x}\n", + pa_addr(unsafe { layout_text_begin() }), + pa_addr(unsafe { layout_text_end() }) + ); + dlog!( + "rodata: {:#x} - {:#x}\n", + pa_addr(unsafe { layout_rodata_begin() }), + pa_addr(unsafe { layout_rodata_end() }) + ); + dlog!( + "data: {:#x} - {:#x}\n", + pa_addr(unsafe { layout_data_begin() }), + pa_addr(unsafe { layout_data_end() }) + ); + + let page_table = + PageTable::new(mpool).map_err(|_| dlog!("Unable to allocate memory for page table.\n")).ok()?; + + // A fake lock. + // TODO(HfO2): IMO, mm_stage1_locked is better to hold a reference to + // PageTable rather than SpinLockGuard>, like + // other similar structures (VmLocked, VCpuExecutionLocked etc.) In that + // case we can delay creating a SpinLock here, and more directly show + // the meaning of unlocked but safe exclusive access of the page table. + let page_table = SpinLock::new(page_table); + let stage1_locked = mm_stage1_locked { + plock: &page_table as *const _ as usize, + }; + + unsafe { + // Let console driver map pages for itself. + plat_console_mm_init(stage1_locked, mpool); + + page_table + .get_mut_unchecked() + .identity_map(layout_text_begin(), layout_text_end(), Mode::X, mpool) + .unwrap(); + page_table + .get_mut_unchecked() + .identity_map(layout_rodata_begin(), layout_rodata_end(), Mode::R, mpool) + .unwrap(); + page_table + .get_mut_unchecked() + .identity_map( + layout_data_begin(), + layout_data_end(), + Mode::R | Mode::W, + mpool, + ) + .unwrap(); + + if !arch_mm_init(page_table.get_unchecked().root, true) { + return None; + } + } + + Some(Self { + HYPERVISOR_PAGE_TABLE: page_table, + STAGE2_INVALIDATE: AtomicBool::new(false), + }) + } +} + /// After calling this function, modifications to stage-2 page tables will use break-before-make and /// invalidate the TLB for the affected range. /// @@ -1032,7 +1100,7 @@ impl<'s> Into>> for mm_stage1_locked { /// This function should not be invoked concurrently with other memory management functions. #[no_mangle] pub unsafe extern "C" fn mm_vm_enable_invalidation() { - STAGE2_INVALIDATE.store(true, Ordering::Relaxed); + MEMORY_MANAGER.get_ref().STAGE2_INVALIDATE.store(true, Ordering::Relaxed); } #[no_mangle] @@ -1162,61 +1230,20 @@ pub unsafe extern "C" fn mm_unmap( stage1_locked.unmap(begin, end, mpool).is_ok() } +/// This function is only used in one unit test (fdt/find_memory_ranges.) +/// Unsafety doesn't really matter. #[no_mangle] pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { - dlog!( - "text: {:#x} - {:#x}\n", - pa_addr(layout_text_begin()), - pa_addr(layout_text_end()) - ); - dlog!( - "rodata: {:#x} - {:#x}\n", - pa_addr(layout_rodata_begin()), - pa_addr(layout_rodata_end()) - ); - dlog!( - "data: {:#x} - {:#x}\n", - pa_addr(layout_data_begin()), - pa_addr(layout_data_end()) - ); - - let mpool = &*mpool; - let page_table = - PageTable::new(mpool).map_err(|_| dlog!("Unable to allocate memory for page table.\n")); - let page_table = ok_or_return!(page_table, false); - let hypervisor_page_table = HYPERVISOR_PAGE_TABLE.get_mut_unchecked(); - ptr::write(hypervisor_page_table, page_table); - - // A fake lock. - let stage1_locked = mm_stage1_locked { - plock: &HYPERVISOR_PAGE_TABLE as *const _ as usize, - }; - - // Let console driver map pages for itself. - plat_console_mm_init(stage1_locked, mpool); - - hypervisor_page_table - .identity_map(layout_text_begin(), layout_text_end(), Mode::X, mpool) - .unwrap(); - hypervisor_page_table - .identity_map(layout_rodata_begin(), layout_rodata_end(), Mode::R, mpool) - .unwrap(); - hypervisor_page_table - .identity_map( - layout_data_begin(), - layout_data_end(), - Mode::R | Mode::W, - mpool, - ) - .unwrap(); + let mm = ok_or_return!(MemoryManager::new(&*mpool).ok_or(()), false); + MEMORY_MANAGER = MaybeUninit::new(mm); - arch_mm_init(hypervisor_page_table.root, true) + true } #[no_mangle] pub unsafe extern "C" fn mm_cpu_init() -> bool { - let ptable = HYPERVISOR_PAGE_TABLE.get_mut_unchecked().root; - arch_mm_init(ptable, false) + let raw_ptable = MEMORY_MANAGER.get_ref().HYPERVISOR_PAGE_TABLE.get_mut_unchecked().root; + arch_mm_init(raw_ptable, false) } #[no_mangle] @@ -1227,7 +1254,8 @@ pub unsafe extern "C" fn mm_defrag(mut stage1_locked: mm_stage1_locked, mpool: * #[no_mangle] pub unsafe extern "C" fn mm_lock_stage1() -> mm_stage1_locked { - HYPERVISOR_PAGE_TABLE.lock().into() + let ptable = &MEMORY_MANAGER.get_ref().HYPERVISOR_PAGE_TABLE; + ptable.lock().into() } #[no_mangle] diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs new file mode 100644 index 000000000..3ddf5ee27 --- /dev/null +++ b/hfo2/src/singleton.rs @@ -0,0 +1,38 @@ +/* + * Copyright 2019 Sanguk Park + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! A module collecting all the singleton objects in Hafnium. +//! +//! This module is dependency-free; Typical solutions of mutable and shared +//! static objects delay their initialization. Considering concurrency, they +//! often use std::sync features to prevent racy initialization. But Hafnium is +//! different. +//! +//! - The initialization is _always_ happened once in the specific time. +//! - During the time, no other thread is running; Hafnium runs as if it were +//! a single-thread program. +//! - After the initialization, Hafnium may make a non-exclusive reference of +//! singletons, but they have their own way for Hafnium to safely write them. +//! +//! Therefore, I do not use a safe wrapper for initialization such as +//! `std::sync::Once` and `lazy_static`. + +use core::mem::MaybeUninit; + +use crate::mm::MemoryManager; + +pub static mut MEMORY_MANAGER: MaybeUninit + = MaybeUninit::uninit(); diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index bf46afa48..7dbe6546a 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -35,6 +35,7 @@ use crate::spci::*; use crate::spinlock::*; use crate::std::*; use crate::types::*; +use crate::singleton::*; const LOG_BUFFER_SIZE: usize = 256; @@ -154,7 +155,9 @@ impl Mailbox { pa_recv_end: paddr_t, local_page_pool: &MPool, ) -> Result<(), ()> { - let mut hypervisor_ptable = HYPERVISOR_PAGE_TABLE.lock(); + // TODO(HfO2): Acquring the singleton here is not recommended. Get the + // hypervisor ptable from callee (API module.) + let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.HYPERVISOR_PAGE_TABLE.lock(); let mut ptable = guard(hypervisor_ptable.deref_mut(), |_| ()); // Map the send page as read-only in the hypervisor address space. From 14c7d6e2331efa854da5b2a7b57755dba26094f7 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 13:09:54 +0900 Subject: [PATCH 15/45] rustfmt. --- hfo2/src/api.rs | 6 ++++-- hfo2/src/mm.rs | 25 ++++++++++++++++++------- hfo2/src/singleton.rs | 11 +++++------ hfo2/src/vm.rs | 6 ++++-- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 0894563fa..481ae1b4a 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -27,6 +27,7 @@ use crate::cpu::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; +use crate::singleton::*; use crate::spci::*; use crate::spci_architected_message::*; use crate::spinlock::*; @@ -34,7 +35,6 @@ use crate::std::*; use crate::types::*; use crate::utils::*; use crate::vm::*; -use crate::singleton::*; // To eliminate the risk of deadlocks, we define a partial order for the acquisition of locks held // concurrently by the same physical CPU. Our current ordering requirements are as follows: @@ -869,7 +869,9 @@ pub unsafe extern "C" fn api_interrupt_inject( /// Clears a region of physical memory by overwriting it with zeros. The data is /// flushed from the cache so the memory has been cleared across the system. fn clear_memory(begin: paddr_t, end: paddr_t, ppool: &MPool) -> Result<(), ()> { - let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.HYPERVISOR_PAGE_TABLE.lock(); + let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() } + .HYPERVISOR_PAGE_TABLE + .lock(); let size = pa_difference(begin, end); let region = pa_addr(begin); diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index c5e694e89..ad1dac455 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -39,10 +39,10 @@ use crate::arch::*; use crate::layout::*; use crate::mpool::MPool; use crate::page::*; +use crate::singleton::*; use crate::spinlock::{SpinLock, SpinLockGuard}; use crate::types::*; use crate::utils::*; -use crate::singleton::*; extern "C" { fn arch_mm_absent_pte(level: u8) -> pte_t; @@ -264,7 +264,10 @@ impl Stage for Stage2 { } fn invalidate_tlb(begin: usize, end: usize) { - if unsafe { MEMORY_MANAGER.get_ref() }.STAGE2_INVALIDATE.load(Ordering::Relaxed) { + if unsafe { MEMORY_MANAGER.get_ref() } + .STAGE2_INVALIDATE + .load(Ordering::Relaxed) + { unsafe { arch_mm_invalidate_stage2_range(ipa_init(begin), ipa_init(end)); } @@ -1044,8 +1047,9 @@ impl MemoryManager { pa_addr(unsafe { layout_data_end() }) ); - let page_table = - PageTable::new(mpool).map_err(|_| dlog!("Unable to allocate memory for page table.\n")).ok()?; + let page_table = PageTable::new(mpool) + .map_err(|_| dlog!("Unable to allocate memory for page table.\n")) + .ok()?; // A fake lock. // TODO(HfO2): IMO, mm_stage1_locked is better to hold a reference to @@ -1057,7 +1061,7 @@ impl MemoryManager { let stage1_locked = mm_stage1_locked { plock: &page_table as *const _ as usize, }; - + unsafe { // Let console driver map pages for itself. plat_console_mm_init(stage1_locked, mpool); @@ -1100,7 +1104,10 @@ impl MemoryManager { /// This function should not be invoked concurrently with other memory management functions. #[no_mangle] pub unsafe extern "C" fn mm_vm_enable_invalidation() { - MEMORY_MANAGER.get_ref().STAGE2_INVALIDATE.store(true, Ordering::Relaxed); + MEMORY_MANAGER + .get_ref() + .STAGE2_INVALIDATE + .store(true, Ordering::Relaxed); } #[no_mangle] @@ -1242,7 +1249,11 @@ pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { #[no_mangle] pub unsafe extern "C" fn mm_cpu_init() -> bool { - let raw_ptable = MEMORY_MANAGER.get_ref().HYPERVISOR_PAGE_TABLE.get_mut_unchecked().root; + let raw_ptable = MEMORY_MANAGER + .get_ref() + .HYPERVISOR_PAGE_TABLE + .get_mut_unchecked() + .root; arch_mm_init(raw_ptable, false) } diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index 3ddf5ee27..fb535cfc6 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -15,24 +15,23 @@ */ //! A module collecting all the singleton objects in Hafnium. -//! +//! //! This module is dependency-free; Typical solutions of mutable and shared //! static objects delay their initialization. Considering concurrency, they //! often use std::sync features to prevent racy initialization. But Hafnium is //! different. -//! +//! //! - The initialization is _always_ happened once in the specific time. //! - During the time, no other thread is running; Hafnium runs as if it were //! a single-thread program. //! - After the initialization, Hafnium may make a non-exclusive reference of //! singletons, but they have their own way for Hafnium to safely write them. -//! +//! //! Therefore, I do not use a safe wrapper for initialization such as -//! `std::sync::Once` and `lazy_static`. +//! `std::sync::Once` and `lazy_static`. use core::mem::MaybeUninit; use crate::mm::MemoryManager; -pub static mut MEMORY_MANAGER: MaybeUninit - = MaybeUninit::uninit(); +pub static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index 7dbe6546a..e8ed62cca 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -31,11 +31,11 @@ use crate::list::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; +use crate::singleton::*; use crate::spci::*; use crate::spinlock::*; use crate::std::*; use crate::types::*; -use crate::singleton::*; const LOG_BUFFER_SIZE: usize = 256; @@ -157,7 +157,9 @@ impl Mailbox { ) -> Result<(), ()> { // TODO(HfO2): Acquring the singleton here is not recommended. Get the // hypervisor ptable from callee (API module.) - let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.HYPERVISOR_PAGE_TABLE.lock(); + let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() } + .HYPERVISOR_PAGE_TABLE + .lock(); let mut ptable = guard(hypervisor_ptable.deref_mut(), |_| ()); // Map the send page as read-only in the hypervisor address space. From 4677f7c752c57e3a1919965074fec1482c362cba Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 13:49:52 +0900 Subject: [PATCH 16/45] Resolve some clippy issues. --- hfo2/src/api.rs | 4 +-- hfo2/src/fdt.rs | 6 ++-- hfo2/src/init.rs | 2 +- hfo2/src/load.rs | 81 +++++++++++++++++++++--------------------------- hfo2/src/mm.rs | 16 +++++----- hfo2/src/vm.rs | 4 +-- 6 files changed, 49 insertions(+), 64 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 481ae1b4a..b10a72067 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -869,9 +869,7 @@ pub unsafe extern "C" fn api_interrupt_inject( /// Clears a region of physical memory by overwriting it with zeros. The data is /// flushed from the cache so the memory has been cleared across the system. fn clear_memory(begin: paddr_t, end: paddr_t, ppool: &MPool) -> Result<(), ()> { - let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() } - .HYPERVISOR_PAGE_TABLE - .lock(); + let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.hypervisor_ptable.lock(); let size = pa_difference(begin, end); let region = pa_addr(begin); diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 7584d6e30..8785b08ce 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -89,7 +89,7 @@ struct FdtTokenizer { } const FDT_VERSION: u32 = 17; -const FDT_MAGIC: u32 = 0xd00dfeed; +const FDT_MAGIC: u32 = 0xd00d_feed; impl FdtTokenizer { fn new(strs: *const u8, begin: *const u8, end: *const u8) -> Self { @@ -192,12 +192,12 @@ impl FdtTokenizer { } match self.str() { - Some(name) => return Some(name), + Some(name) => Some(name), None => { // Move cursor to the end so that caller won't get any new // tokens. self.cur = self.end; - return None; + None } } } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index ace6bc409..67327ad2a 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -63,7 +63,7 @@ unsafe fn one_time_init() { dlog_enable_lock(); mpool_enable_locks(); - let mut hypervisor_ptable = MEMORY_MANAGER.get_ref().HYPERVISOR_PAGE_TABLE.lock(); + let mut hypervisor_ptable = MEMORY_MANAGER.get_ref().hypervisor_ptable.lock(); let mut params = boot_params::get(&mut hypervisor_ptable, &mut ppool) .unwrap_or_else(|| panic!("unable to retrieve boot params")); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index baaf586da..853caee66 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -33,6 +33,8 @@ use crate::types::*; use crate::utils::*; use crate::vm::*; +use arrayvec::ArrayVec; + /// Copies data to an unmapped location by mapping it for write, copying the /// data, then unmapping it. /// @@ -58,7 +60,7 @@ unsafe fn copy_to_unmapped( memcpy_s(pa_addr(to) as *mut _, size, from, size); arch_mm_write_back_dcache(pa_addr(to), size); - hypervisor_ptable.unmap(to, to_end, ppool); + hypervisor_ptable.unmap(to, to_end, ppool).unwrap(); true } @@ -144,29 +146,24 @@ pub unsafe fn load_primary( /// Try to find a memory range of the given size within the given ranges, and /// remove it from them. Return true on success, or false if no large enough /// contiguous range is found. -unsafe fn carve_out_mem_range( - mem_ranges: *mut MemRange, - mem_ranges_count: usize, +fn carve_out_mem_range( + mem_ranges: &mut [MemRange], size_to_find: u64, - found_begin: *mut paddr_t, - found_end: *mut paddr_t, -) -> bool { +) -> Result<(paddr_t, paddr_t), ()> { // TODO(b/116191358): Consider being cleverer about how we pack VMs // together, with a non-greedy algorithm. - for i in 0..mem_ranges_count { - if size_to_find - <= pa_difference((*mem_ranges.add(i)).begin, (*mem_ranges.add(i)).end) as u64 - { + for mem_range in mem_ranges.iter_mut() { + if size_to_find <= pa_difference(mem_range.begin, mem_range.end) as u64 { // This range is big enough, take some of it from the end and reduce // its size accordingly. - *found_end = (*mem_ranges.add(i)).end; - *found_begin = pa_init(pa_addr((*mem_ranges.add(i)).end) - size_to_find as usize); - (*mem_ranges.add(i)).end = *found_begin; - return true; + let found_end = mem_range.end; + let found_begin = pa_init(pa_addr(mem_range.end) - size_to_find as usize); + mem_range.end = found_begin; + return Ok((found_begin, found_end)); } } - false + Err(()) } /// Given arrays of memory ranges before and after memory was removed for @@ -175,13 +172,11 @@ unsafe fn carve_out_mem_range( /// MAX_MEM_RANGES reserved ranges after adding the new ones. /// `before` and `after` must be arrays of exactly `mem_ranges_count` elements. unsafe fn update_reserved_ranges( - update: *mut BootParamsUpdate, - before: *const MemRange, - after: *const MemRange, - mem_ranges_count: usize, + update: &mut BootParamsUpdate, + before: &[MemRange], + after: &[MemRange], ) -> Result<(), ()> { - let after = slice::from_raw_parts(after, mem_ranges_count); - let before = slice::from_raw_parts(before, mem_ranges_count); + assert_eq!(before.len(), after.len()); for (before, after) in before.iter().zip(after.iter()) { if pa_addr(after.begin) > pa_addr(before.begin) { @@ -190,9 +185,9 @@ unsafe fn update_reserved_ranges( return Err(()); } - (*update).reserved_ranges[(*update).reserved_ranges_count].begin = after.end; - (*update).reserved_ranges[(*update).reserved_ranges_count].end = before.end; - (*update).reserved_ranges_count += 1; + update.reserved_ranges[update.reserved_ranges_count].begin = after.end; + update.reserved_ranges[update.reserved_ranges_count].end = before.end; + update.reserved_ranges_count += 1; } } @@ -205,16 +200,17 @@ pub unsafe fn load_secondary( hypervisor_ptable: &mut PageTable, cpio: *const MemIter, params: *const BootParams, - update: *mut BootParamsUpdate, + update: &mut BootParamsUpdate, ppool: &mut MPool, ) -> Result<(), ()> { - let mut mem_ranges_available: [MemRange; MAX_MEM_RANGES] = mem::uninitialized(); + let mut mem_ranges_available: ArrayVec<[MemRange; MAX_MEM_RANGES]> = ArrayVec::new(); // static_assert( // sizeof(mem_ranges_available) == sizeof(params->mem_ranges), // "mem_range arrays must be the same size for memcpy."); const_assert!(mem::size_of::() * MAX_MEM_RANGES < 500); mem_ranges_available.clone_from_slice(&(*params).mem_ranges); + mem_ranges_available.truncate((*params).mem_ranges_count); let primary = vm_find(HF_PRIMARY_VM_ID); let mut it = mem::uninitialized(); @@ -225,9 +221,8 @@ pub unsafe fn load_secondary( } // Round the last addresses down to the page size. - for i in 0..(*params).mem_ranges_count { - mem_ranges_available[i].end = - pa_init(round_down(pa_addr(mem_ranges_available[i].end), PAGE_SIZE)); + for mem_range in mem_ranges_available.iter_mut() { + mem_range.end = pa_init(round_down(pa_addr(mem_range.end), PAGE_SIZE)); } let mut mem = mem::uninitialized(); @@ -261,19 +256,14 @@ pub unsafe fn load_secondary( continue; } - let mut secondary_mem_begin = mem::uninitialized(); - let mut secondary_mem_end = mem::uninitialized(); - - if !carve_out_mem_range( - mem_ranges_available.as_mut_ptr(), - (*params).mem_ranges_count, - mem, - &mut secondary_mem_begin, - &mut secondary_mem_end, - ) { - dlog!("Not enough memory ({} bytes)\n", mem); - continue; - } + let (secondary_mem_begin, secondary_mem_end) = + match carve_out_mem_range(&mut mem_ranges_available, mem as u64) { + Ok(range) => range, + Err(_) => { + dlog!("Not enough memory ({} bytes)\n", mem); + continue; + } + }; if !copy_to_unmapped( hypervisor_ptable, @@ -339,8 +329,7 @@ pub unsafe fn load_secondary( // smaller. update_reserved_ranges( update, - (*params).mem_ranges.as_ptr(), - mem_ranges_available.as_ptr(), - (*params).mem_ranges_count, + &(*params).mem_ranges[0..(*params).mem_ranges_count], + &mem_ranges_available, ) } diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index ad1dac455..5d7523f1b 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -265,7 +265,7 @@ impl Stage for Stage2 { fn invalidate_tlb(begin: usize, end: usize) { if unsafe { MEMORY_MANAGER.get_ref() } - .STAGE2_INVALIDATE + .stage2_invalidate .load(Ordering::Relaxed) { unsafe { @@ -1023,10 +1023,10 @@ impl<'s> Into>> for mm_stage1_locked { pub struct MemoryManager { /// The hypervisor page table. - pub HYPERVISOR_PAGE_TABLE: SpinLock>, + pub hypervisor_ptable: SpinLock>, /// Is stage2 invalidation enabled? - pub STAGE2_INVALIDATE: AtomicBool, + pub stage2_invalidate: AtomicBool, } impl MemoryManager { @@ -1090,8 +1090,8 @@ impl MemoryManager { } Some(Self { - HYPERVISOR_PAGE_TABLE: page_table, - STAGE2_INVALIDATE: AtomicBool::new(false), + hypervisor_ptable: page_table, + stage2_invalidate: AtomicBool::new(false), }) } } @@ -1106,7 +1106,7 @@ impl MemoryManager { pub unsafe extern "C" fn mm_vm_enable_invalidation() { MEMORY_MANAGER .get_ref() - .STAGE2_INVALIDATE + .stage2_invalidate .store(true, Ordering::Relaxed); } @@ -1251,7 +1251,7 @@ pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { pub unsafe extern "C" fn mm_cpu_init() -> bool { let raw_ptable = MEMORY_MANAGER .get_ref() - .HYPERVISOR_PAGE_TABLE + .hypervisor_ptable .get_mut_unchecked() .root; arch_mm_init(raw_ptable, false) @@ -1265,7 +1265,7 @@ pub unsafe extern "C" fn mm_defrag(mut stage1_locked: mm_stage1_locked, mpool: * #[no_mangle] pub unsafe extern "C" fn mm_lock_stage1() -> mm_stage1_locked { - let ptable = &MEMORY_MANAGER.get_ref().HYPERVISOR_PAGE_TABLE; + let ptable = &MEMORY_MANAGER.get_ref().hypervisor_ptable; ptable.lock().into() } diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index e8ed62cca..de278a47c 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -157,9 +157,7 @@ impl Mailbox { ) -> Result<(), ()> { // TODO(HfO2): Acquring the singleton here is not recommended. Get the // hypervisor ptable from callee (API module.) - let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() } - .HYPERVISOR_PAGE_TABLE - .lock(); + let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.hypervisor_ptable.lock(); let mut ptable = guard(hypervisor_ptable.deref_mut(), |_| ()); // Map the send page as read-only in the hypervisor address space. From 4b85b64af13dd44bf08eca9130955d215e59e2f6 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 14:09:52 +0900 Subject: [PATCH 17/45] Fix mem_ranges_available to be initialized properly --- hfo2/src/load.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 853caee66..41ea530a1 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -16,7 +16,6 @@ use core::mem::{self, MaybeUninit}; use core::ptr; -use core::slice; use crate::addr::*; use crate::arch::*; @@ -180,7 +179,7 @@ unsafe fn update_reserved_ranges( for (before, after) in before.iter().zip(after.iter()) { if pa_addr(after.begin) > pa_addr(before.begin) { - if (*update).reserved_ranges_count >= MAX_MEM_RANGES { + if update.reserved_ranges_count >= MAX_MEM_RANGES { dlog!("Too many reserved ranges after loading secondary VMs.\n"); return Err(()); } @@ -198,8 +197,8 @@ unsafe fn update_reserved_ranges( /// Memory reserved for the VMs is added to the `reserved_ranges` of `update`. pub unsafe fn load_secondary( hypervisor_ptable: &mut PageTable, - cpio: *const MemIter, - params: *const BootParams, + cpio: &MemIter, + params: &BootParams, update: &mut BootParamsUpdate, ppool: &mut MPool, ) -> Result<(), ()> { @@ -209,8 +208,9 @@ pub unsafe fn load_secondary( // "mem_range arrays must be the same size for memcpy."); const_assert!(mem::size_of::() * MAX_MEM_RANGES < 500); - mem_ranges_available.clone_from_slice(&(*params).mem_ranges); - mem_ranges_available.truncate((*params).mem_ranges_count); + mem_ranges_available.set_len(MAX_MEM_RANGES); + mem_ranges_available.clone_from_slice(¶ms.mem_ranges); + mem_ranges_available.truncate(params.mem_ranges_count); let primary = vm_find(HF_PRIMARY_VM_ID); let mut it = mem::uninitialized(); @@ -329,7 +329,7 @@ pub unsafe fn load_secondary( // smaller. update_reserved_ranges( update, - &(*params).mem_ranges[0..(*params).mem_ranges_count], + ¶ms.mem_ranges[0..params.mem_ranges_count], &mem_ranges_available, ) } From fc876867ff4a7cf716ba66766128ca951d90f178 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 16:39:16 +0900 Subject: [PATCH 18/45] Refactor VM init sequence. --- hfo2/src/init.rs | 2 + hfo2/src/load.rs | 49 +++++++++++--------- hfo2/src/singleton.rs | 2 + hfo2/src/vm.rs | 103 +++++++++++++++++++++++++----------------- 4 files changed, 94 insertions(+), 62 deletions(-) diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 67327ad2a..27ea74cd9 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -101,6 +101,8 @@ unsafe fn one_time_init() { pa_difference(params.initrd_begin, params.initrd_end), ); + VM_MANAGER = MaybeUninit::new(VmManager::new()); + // Load all VMs. let primary_initrd = load_primary(&mut hypervisor_ptable, &cpio, params.kernel_arg, &mut ppool) .unwrap_or_else(|_| panic!("unable to load primary VM")); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 41ea530a1..9b8b434dd 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -27,6 +27,7 @@ use crate::memiter::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; +use crate::singleton::*; use crate::std::*; use crate::types::*; use crate::utils::*; @@ -45,7 +46,7 @@ unsafe fn copy_to_unmapped( to: paddr_t, from: *const c_void, size: usize, - ppool: &mut MPool, + ppool: &MPool, ) -> bool { let to_end = pa_add(to, size); @@ -67,9 +68,9 @@ unsafe fn copy_to_unmapped( /// Loads the primary VM. pub unsafe fn load_primary( hypervisor_ptable: &mut PageTable, - cpio: *const MemIter, + cpio: &MemIter, kernel_arg: uintreg_t, - ppool: &mut MPool, + ppool: &MPool, ) -> Result { let mut it = mem::uninitialized(); let primary_begin = layout_primary_begin(); @@ -103,27 +104,33 @@ pub unsafe fn load_primary( let initrd = initrd.assume_init(); { - let mut vm = mem::uninitialized(); - if !vm_init(MAX_CPUS as u16, ppool, &mut vm) { - dlog!("Unable to initialise primary vm\n"); - return Err(()); - } - - if (*vm).id != HF_PRIMARY_VM_ID { + let vm = VM_MANAGER + .get_mut() + .new_vm(MAX_CPUS as spci_vcpu_count_t, ppool) + .ok_or_else(|| { + dlog!("Unable to initialise primary vm\n"); + (()) + })?; + + if vm.id != HF_PRIMARY_VM_ID { dlog!("Primary vm was not given correct id\n"); return Err(()); } // Map the 1TB of memory. // TODO: We should do a whitelist rather than blacklist. - if !mm_vm_identity_map( - &mut (*vm).inner.get_mut_unchecked().ptable, - pa_init(0), - pa_init(1024usize * 1024 * 1024 * 1024), - Mode::R | Mode::W | Mode::X, - ptr::null_mut(), - ppool, - ) { + if vm + .inner + .get_mut() + .ptable + .identity_map( + pa_init(0), + pa_init(1024usize * 1024 * 1024 * 1024), + Mode::R | Mode::W | Mode::X, + ppool, + ) + .is_err() + { dlog!("Unable to initialise memory for primary vm\n"); return Err(()); } @@ -133,9 +140,9 @@ pub unsafe fn load_primary( return Err(()); } - (*vm_get_vcpu(vm, 0)) + vm.vcpus[0] .inner - .lock() + .lock() // TODO(HfO2): We can safely use get_mut() here .on(ipa_from_pa(primary_begin), kernel_arg); } @@ -170,7 +177,7 @@ fn carve_out_mem_range( /// update. Return true on success, or false if there would be more than /// MAX_MEM_RANGES reserved ranges after adding the new ones. /// `before` and `after` must be arrays of exactly `mem_ranges_count` elements. -unsafe fn update_reserved_ranges( +fn update_reserved_ranges( update: &mut BootParamsUpdate, before: &[MemRange], after: &[MemRange], diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index fb535cfc6..85a094d4f 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -33,5 +33,7 @@ use core::mem::MaybeUninit; use crate::mm::MemoryManager; +use crate::vm::VmManager; pub static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); +pub static mut VM_MANAGER: MaybeUninit = MaybeUninit::uninit(); diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index de278a47c..09dd52f33 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -219,7 +219,7 @@ pub struct VmInner { impl VmInner { /// Initializes VmInner. - pub unsafe fn init(&mut self, vm: *mut Vm, ppool: &mut MPool) -> Result<(), ()> { + pub unsafe fn init(&mut self, vm: *mut Vm, ppool: &MPool) -> Result<(), ()> { self.mailbox.init(); if !mm_vm_init(&mut self.ptable, ppool) { @@ -480,6 +480,26 @@ pub struct Vm { } impl Vm { + pub fn init( + &mut self, + id: spci_vm_id_t, + vcpu_count: spci_vcpu_count_t, + ppool: &MPool, + ) -> Result<(), ()> { + self.id = id; + self.vcpu_count = vcpu_count; + self.aborting = AtomicBool::new(false); + unsafe { + let self_ptr = self as *mut _; + self.inner.get_mut().init(self_ptr, ppool); + + for i in 0..vcpu_count { + vcpu_init(vm_get_vcpu(self, i), self); + } + } + Ok(()) + } + /// Returns the root address of the page table of this VM. It is safe not to /// lock `self.inner` because the value of `ptable.as_raw()` doesn't change /// after `ptable` is initialized. Of course, actual page table may vary @@ -542,62 +562,63 @@ impl VmLocked { } } -static mut VMS: MaybeUninit<[Vm; MAX_VMS]> = MaybeUninit::uninit(); -static mut VM_COUNT: spci_vm_count_t = 0; - -#[no_mangle] -pub unsafe extern "C" fn vm_init( - vcpu_count: spci_vcpu_count_t, - ppool: *mut MPool, - new_vm: *mut *mut Vm, -) -> bool { - let vm: *mut Vm; +pub struct VmManager { + vms: ArrayVec<[Vm; MAX_VMS]>, +} - if VM_COUNT as usize >= MAX_VMS { - return false; +impl VmManager { + pub fn new() -> Self { + Self { + vms: ArrayVec::new(), + } } - vm = &mut VMS.get_mut()[VM_COUNT as usize]; + pub fn new_vm(&mut self, vcpu_count: spci_vcpu_count_t, ppool: &MPool) -> Option<&mut Vm> { + if self.vms.is_full() { + return None; + } - memset_s( - vm as usize as _, - mem::size_of::(), - 0, - mem::size_of::(), - ); + let id = self.vms.len(); + let vm = unsafe { self.vms.get_unchecked_mut(id) }; - (*vm).id = VM_COUNT; - (*vm).vcpu_count = vcpu_count; - (*vm).aborting = AtomicBool::new(false); - let result = (*vm).inner.get_mut_unchecked().init(vm, &mut *ppool); - if result.is_err() { - return false; - } + vm.init(id as u16, vcpu_count, ppool).ok()?; - // Do basic initialization of vcpus. - for i in 0..vcpu_count { - vcpu_init(vm_get_vcpu(vm, i), vm); - } + unsafe { + self.vms.set_len(id + 1); + } - VM_COUNT += 1; - *new_vm = vm; + Some(&mut self.vms[id]) + } +} - true +#[no_mangle] +pub unsafe extern "C" fn vm_init( + vcpu_count: spci_vcpu_count_t, + ppool: *mut MPool, + new_vm: *mut *mut Vm, +) -> bool { + match VM_MANAGER.get_mut().new_vm(vcpu_count, &mut *ppool) { + Some(vm) => { + *new_vm = vm as *mut _; + true + } + None => false, + } } #[no_mangle] pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { - VM_COUNT + VM_MANAGER.get_ref().vms.len() as spci_vm_count_t } #[no_mangle] pub unsafe extern "C" fn vm_find(id: spci_vm_id_t) -> *mut Vm { - // Ensure the VM is initialized. - if id >= VM_COUNT { - return ptr::null_mut(); - } - - &mut VMS.get_mut()[id as usize] + VM_MANAGER + .get_mut() + .vms + .get_mut(id as usize) // Ensure the VM is initialized. + .map(|vm| vm as *mut _) + .unwrap_or(ptr::null_mut()) } /// Locks the given VM and updates `locked` to hold the newly locked vm. From 896d267323963b7dc03af4032d4cc9515c71c9f2 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 17:15:00 +0900 Subject: [PATCH 19/45] Refactor load_secondary. --- hfo2/src/load.rs | 36 +++++++++++++++++++----------------- hfo2/src/vm.rs | 8 ++++++++ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 9b8b434dd..a8b524bdd 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -15,7 +15,6 @@ */ use core::mem::{self, MaybeUninit}; -use core::ptr; use crate::addr::*; use crate::arch::*; @@ -219,7 +218,7 @@ pub unsafe fn load_secondary( mem_ranges_available.clone_from_slice(¶ms.mem_ranges); mem_ranges_available.truncate(params.mem_ranges_count); - let primary = vm_find(HF_PRIMARY_VM_ID); + let primary = VM_MANAGER.get_mut().get_mut(HF_PRIMARY_VM_ID).ok_or(())?; let mut it = mem::uninitialized(); if !cpio_find_file(cpio, "vms.txt\0".as_ptr(), &mut it) { @@ -283,34 +282,38 @@ pub unsafe fn load_secondary( continue; } - let mut vm = mem::uninitialized(); - if !vm_init(cpu as spci_vcpu_count_t, ppool, &mut vm) { - dlog!("Unable to initialise VM\n"); - continue; - } + // TODO(HfO2): This should be rejected by borrowck, IMO. (by defining + // primary, VM_MANAGER is borrowed exclusively before.) Maybe unsafe + // static mut confused borrowck? + let vm = match VM_MANAGER + .get_mut() + .new_vm(cpu as spci_vcpu_count_t, ppool) { + Some(vm) => vm, + None => { + dlog!("Unable to initialise VM\n"); + continue; + } + }; - let mut secondary_entry = mem::uninitialized(); + let secondary_entry = ipa_from_pa(secondary_mem_begin); // Grant the VM access to the memory. - if !mm_vm_identity_map( - &mut (*vm).inner.get_mut_unchecked().ptable, + if vm.inner.get_mut().ptable.identity_map( secondary_mem_begin, secondary_mem_end, Mode::R | Mode::W | Mode::X, - &mut secondary_entry, ppool, - ) { + ).is_err() { dlog!("Unable to initialise memory\n"); continue; } // Deny the primary VM access to this memory. - if !mm_vm_unmap( - &mut (*primary).inner.get_mut_unchecked().ptable, + if primary.inner.get_mut().ptable.unmap( secondary_mem_begin, secondary_mem_end, ppool, - ) { + ).is_err() { dlog!("Unable to unmap secondary VM from primary VM\n"); return Err(()); } @@ -321,9 +324,8 @@ pub unsafe fn load_secondary( pa_addr(secondary_mem_begin) ); - let vcpu = vm_get_vcpu(vm, 0); vcpu_secondary_reset_and_start( - vcpu, + &mut vm.vcpus[0], secondary_entry, pa_difference(secondary_mem_begin, secondary_mem_end) as uintreg_t, ); diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index 09dd52f33..c65cb7210 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -589,6 +589,14 @@ impl VmManager { Some(&mut self.vms[id]) } + + pub fn get(&self, id: spci_vm_id_t) -> Option<&Vm> { + self.vms.get(id as usize) + } + + pub fn get_mut(&mut self, id: spci_vm_id_t) -> Option<&mut Vm> { + self.vms.get_mut(id as usize) + } } #[no_mangle] From be86a722e9ac720644c0a477cbda173f887a4a92 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 17:49:55 +0900 Subject: [PATCH 20/45] Refactor VM init sequence. (cont'd) --- hfo2/src/api.rs | 6 +++--- hfo2/src/cpu.rs | 34 ++++++++++++++++++++++++++++++++++ hfo2/src/vm.rs | 15 +++++++-------- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index b10a72067..bc90fd61a 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -238,7 +238,7 @@ pub unsafe extern "C" fn api_vcpu_get_count( return 0; } - (*vm).vcpu_count + (*vm).vcpus.len() as spci_vcpu_count_t } /// This function is called by the architecture-specific context switching @@ -402,7 +402,7 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested vcpu must exist. - if vcpu_idx >= (*vm).vcpu_count { + if vcpu_idx as usize >= (*vm).vcpus.len() { return ret.into_raw(); } @@ -844,7 +844,7 @@ pub unsafe extern "C" fn api_interrupt_inject( return -1; } - if target_vcpu_idx >= (*target_vm).vcpu_count { + if target_vcpu_idx as usize >= (*target_vm).vcpus.len() { // The requested vcpu must exist. return -1; } diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 38b450af9..1c8741467 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -97,6 +97,13 @@ pub struct Interrupts { } impl Interrupts { + pub fn new() -> Self { + Self { + enabled: [0; HF_NUM_INTIDS as usize / INTERRUPT_REGISTER_BITS], + pending: [0; HF_NUM_INTIDS as usize / INTERRUPT_REGISTER_BITS], + enabled_and_pending_count: 0, + } + } pub fn id_to_index(intid: intid_t) -> Result<(usize, u32), ()> { if intid >= HF_NUM_INTIDS { return Err(()); @@ -191,6 +198,15 @@ impl Interrupts { } impl ArchRegs { + pub fn new() -> Self { + let mut ret; + unsafe { + ret = MaybeUninit::uninit().assume_init(); + memset_s(&mut ret as *mut _ as usize as *mut _, mem::size_of_val(&ret), 0, mem::size_of_val(&ret)); + } + ret + } + /// Reset the register values other than the PC and argument which are set /// with `arch_regs_set_pc_arg()`. pub fn reset(&mut self, is_primary: bool, vm: &Vm, vcpu_id: cpu_id_t) { @@ -243,6 +259,14 @@ pub struct VCpuInner { } impl VCpuInner { + pub fn new() -> Self { + Self { + state: VCpuStatus::Off, + cpu: ptr::null_mut(), + regs: ArchRegs::new(), + } + } + /// Initialise the registers for the given vCPU and set the state to /// VCpuStatus::Ready. The caller must hold the vCPU execution lock while /// calling this. @@ -270,6 +294,16 @@ pub struct VCpu { pub interrupts: SpinLock, } +impl VCpu { + pub fn new(vm: *mut Vm) -> Self { + Self { + vm, + inner: SpinLock::new(VCpuInner::new()), + interrupts: SpinLock::new(Interrupts::new()), + } + } +} + /// Encapsulates a vCPU whose lock is held. #[repr(C)] pub struct VCpuExecutionLocked { diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index c65cb7210..4d56ca071 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -464,7 +464,6 @@ impl VmInner { pub struct Vm { pub id: spci_vm_id_t, - pub vcpu_count: spci_vcpu_count_t, /// VCpus of this vm. /// Note: This field is regarded as a kind of mutable states of Vm, but is @@ -472,7 +471,7 @@ pub struct Vm { /// 1. Mutable inner fields are contained in VCpuState. /// 2. VCpuState has higher lock order than one of Vm. It is nonsense to /// lock VmInner to acquire VCpuState. - pub vcpus: [VCpu; MAX_CPUS], + pub vcpus: ArrayVec<[VCpu; MAX_CPUS]>, /// See api.c for the partial ordering on locks. pub inner: SpinLock, @@ -487,14 +486,14 @@ impl Vm { ppool: &MPool, ) -> Result<(), ()> { self.id = id; - self.vcpu_count = vcpu_count; + self.vcpus = ArrayVec::new(); self.aborting = AtomicBool::new(false); unsafe { let self_ptr = self as *mut _; - self.inner.get_mut().init(self_ptr, ppool); + self.inner.get_mut().init(self_ptr, ppool)?; - for i in 0..vcpu_count { - vcpu_init(vm_get_vcpu(self, i), self); + for _ in 0..vcpu_count { + self.vcpus.push(VCpu::new(self_ptr)); } } Ok(()) @@ -648,7 +647,7 @@ pub unsafe extern "C" fn vm_unlock(locked: *mut VmLocked) { /// This assumes the index is valid, i.e. less than vm->vcpu_count. #[no_mangle] pub unsafe extern "C" fn vm_get_vcpu(vm: *mut Vm, vcpu_index: spci_vcpu_index_t) -> *mut VCpu { - assert!(vcpu_index < (*vm).vcpu_count); + assert!((vcpu_index as usize) < (*vm).vcpus.len()); &mut (*vm).vcpus[vcpu_index as usize] } @@ -669,5 +668,5 @@ pub unsafe extern "C" fn vm_get_arch(vm: *mut Vm) -> *mut ArchVm { #[no_mangle] pub unsafe extern "C" fn vm_get_vcpu_count(vm: *const Vm) -> spci_vcpu_count_t { - (*vm).vcpu_count + (*vm).vcpus.len() as spci_vcpu_count_t } From 16c7781e8a42293158b3e2afc6e1c5621262501b Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 18:12:42 +0900 Subject: [PATCH 21/45] Refactor API init sequence. --- hfo2/src/api.rs | 28 +++++------ hfo2/src/cpu.rs | 7 ++- hfo2/src/init.rs | 4 +- hfo2/src/load.rs | 109 ++++++++++++++++++++++-------------------- hfo2/src/singleton.rs | 2 + hfo2/src/spci.rs | 4 +- hfo2/src/vm.rs | 5 +- inc/hf/api.h | 1 - 8 files changed, 85 insertions(+), 75 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index bc90fd61a..87fbb3cf4 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -48,17 +48,17 @@ use crate::vm::*; // of a page. const_assert_eq!(hf_mailbox_size; HF_MAILBOX_SIZE, PAGE_SIZE); -/// A global page pool for sharing memories. Its mutability is needed only for -/// initialization. -static mut API_PAGE_POOL: MaybeUninit = MaybeUninit::uninit(); - -/// Initialises the API page pool by taking ownership of the contents of the -/// given page pool. -/// TODO(HfO2): The ownership of `ppool` is actually moved from `one_time_init` -/// to here. Refactor this function like `Api::new(ppool: MPool) -> Api`. (#31) -#[no_mangle] -pub unsafe extern "C" fn api_init(ppool: *const MPool) { - API_PAGE_POOL = MaybeUninit::new(MPool::new_from(&*ppool)); +pub struct ApiManager { + /// A global page pool for sharing memories. + ppool: MPool, +} + +impl ApiManager { + /// Initialises the API page pool by taking ownership of the contents of the + /// given page pool. + pub fn new(ppool: MPool) -> Self { + Self { ppool } + } } /// Switches the physical CPU back to the corresponding vcpu of the primary VM. @@ -495,7 +495,7 @@ pub unsafe extern "C" fn api_vm_configure( // to keep a single unlock point. let mut vm_inner = (*vm).inner.lock(); if vm_inner - .configure(send, recv, API_PAGE_POOL.get_ref()) + .configure(send, recv, &API_MANAGER.get_ref().ppool) .is_err() { return -1; @@ -926,7 +926,7 @@ pub fn spci_share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool: MPool = MPool::new_with_fallback(unsafe { API_PAGE_POOL.get_ref() }); + let local_page_pool: MPool = MPool::new_with_fallback(unsafe { &API_MANAGER.get_ref().ppool }); // Obtain the single contiguous set of pages from the memory_region. // TODO: Add support for multiple constituent regions. @@ -1037,7 +1037,7 @@ fn share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool = MPool::new_with_fallback(unsafe { API_PAGE_POOL.get_ref() }); + let local_page_pool = MPool::new_with_fallback(unsafe { &API_MANAGER.get_ref().ppool }); let (mut from_inner, mut to_inner) = SpinLock::lock_both(&(*from).inner, &(*to).inner); diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 1c8741467..3cf964b7c 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -202,7 +202,12 @@ impl ArchRegs { let mut ret; unsafe { ret = MaybeUninit::uninit().assume_init(); - memset_s(&mut ret as *mut _ as usize as *mut _, mem::size_of_val(&ret), 0, mem::size_of_val(&ret)); + memset_s( + &mut ret as *mut _ as usize as *mut _, + mem::size_of_val(&ret), + 0, + mem::size_of_val(&ret), + ); } ret } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 27ea74cd9..54467bd79 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -104,7 +104,7 @@ unsafe fn one_time_init() { VM_MANAGER = MaybeUninit::new(VmManager::new()); // Load all VMs. - let primary_initrd = load_primary(&mut hypervisor_ptable, &cpio, params.kernel_arg, &mut ppool) + let primary_initrd = load_primary(&mut hypervisor_ptable, &cpio, params.kernel_arg, &ppool) .unwrap_or_else(|_| panic!("unable to load primary VM")); // load_secondary will add regions assigned to the secondary VMs from @@ -134,7 +134,7 @@ unsafe fn one_time_init() { hypervisor_ptable.defrag(&ppool); // Initialise the API page pool. ppool will be empty from now on. - api_init(&ppool); + API_MANAGER = MaybeUninit::new(ApiManager::new(ppool)); // Enable TLB invalidation for VM page table updates. mm_vm_enable_invalidation(); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index a8b524bdd..8f4fb09aa 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -30,7 +30,6 @@ use crate::singleton::*; use crate::std::*; use crate::types::*; use crate::utils::*; -use crate::vm::*; use arrayvec::ArrayVec; @@ -102,49 +101,47 @@ pub unsafe fn load_primary( let initrd = initrd.assume_init(); - { - let vm = VM_MANAGER - .get_mut() - .new_vm(MAX_CPUS as spci_vcpu_count_t, ppool) - .ok_or_else(|| { - dlog!("Unable to initialise primary vm\n"); - (()) - })?; - - if vm.id != HF_PRIMARY_VM_ID { - dlog!("Primary vm was not given correct id\n"); - return Err(()); - } + let vm = VM_MANAGER + .get_mut() + .new_vm(MAX_CPUS as spci_vcpu_count_t, ppool) + .ok_or_else(|| { + dlog!("Unable to initialise primary vm\n"); + (()) + })?; - // Map the 1TB of memory. - // TODO: We should do a whitelist rather than blacklist. - if vm - .inner - .get_mut() - .ptable - .identity_map( - pa_init(0), - pa_init(1024usize * 1024 * 1024 * 1024), - Mode::R | Mode::W | Mode::X, - ppool, - ) - .is_err() - { - dlog!("Unable to initialise memory for primary vm\n"); - return Err(()); - } + if vm.id != HF_PRIMARY_VM_ID { + dlog!("Primary vm was not given correct id\n"); + return Err(()); + } - if !mm_vm_unmap_hypervisor(&mut (*vm).inner.get_mut_unchecked().ptable, ppool) { - dlog!("Unable to unmap hypervisor from primary vm\n"); - return Err(()); - } + // Map the 1TB of memory. + // TODO: We should do a whitelist rather than blacklist. + if vm + .inner + .get_mut() + .ptable + .identity_map( + pa_init(0), + pa_init(1024usize * 1024 * 1024 * 1024), + Mode::R | Mode::W | Mode::X, + ppool, + ) + .is_err() + { + dlog!("Unable to initialise memory for primary vm\n"); + return Err(()); + } - vm.vcpus[0] - .inner - .lock() // TODO(HfO2): We can safely use get_mut() here - .on(ipa_from_pa(primary_begin), kernel_arg); + if !mm_vm_unmap_hypervisor(&mut (*vm).inner.get_mut_unchecked().ptable, ppool) { + dlog!("Unable to unmap hypervisor from primary vm\n"); + return Err(()); } + vm.vcpus[0] + .inner + .lock() // TODO(HfO2): We can safely use get_mut() here + .on(ipa_from_pa(primary_begin), kernel_arg); + Ok(initrd) } @@ -285,9 +282,7 @@ pub unsafe fn load_secondary( // TODO(HfO2): This should be rejected by borrowck, IMO. (by defining // primary, VM_MANAGER is borrowed exclusively before.) Maybe unsafe // static mut confused borrowck? - let vm = match VM_MANAGER - .get_mut() - .new_vm(cpu as spci_vcpu_count_t, ppool) { + let vm = match VM_MANAGER.get_mut().new_vm(cpu as spci_vcpu_count_t, ppool) { Some(vm) => vm, None => { dlog!("Unable to initialise VM\n"); @@ -298,22 +293,30 @@ pub unsafe fn load_secondary( let secondary_entry = ipa_from_pa(secondary_mem_begin); // Grant the VM access to the memory. - if vm.inner.get_mut().ptable.identity_map( - secondary_mem_begin, - secondary_mem_end, - Mode::R | Mode::W | Mode::X, - ppool, - ).is_err() { + if vm + .inner + .get_mut() + .ptable + .identity_map( + secondary_mem_begin, + secondary_mem_end, + Mode::R | Mode::W | Mode::X, + ppool, + ) + .is_err() + { dlog!("Unable to initialise memory\n"); continue; } // Deny the primary VM access to this memory. - if primary.inner.get_mut().ptable.unmap( - secondary_mem_begin, - secondary_mem_end, - ppool, - ).is_err() { + if primary + .inner + .get_mut() + .ptable + .unmap(secondary_mem_begin, secondary_mem_end, ppool) + .is_err() + { dlog!("Unable to unmap secondary VM from primary VM\n"); return Err(()); } diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index 85a094d4f..f6f0bc4af 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -32,8 +32,10 @@ use core::mem::MaybeUninit; +use crate::api::ApiManager; use crate::mm::MemoryManager; use crate::vm::VmManager; pub static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); pub static mut VM_MANAGER: MaybeUninit = MaybeUninit::uninit(); +pub static mut API_MANAGER: MaybeUninit = MaybeUninit::uninit(); diff --git a/hfo2/src/spci.rs b/hfo2/src/spci.rs index 08d55bea0..4563f2b35 100644 --- a/hfo2/src/spci.rs +++ b/hfo2/src/spci.rs @@ -132,7 +132,9 @@ impl SpciMessage { #[inline] pub fn get_architected_message_header(&self) -> &SpciArchitectedMessageHeader { #[allow(clippy::cast_ptr_alignment)] - unsafe { &*(self.payload.as_ptr() as *const _) } + unsafe { + &*(self.payload.as_ptr() as *const _) + } } } diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index 4d56ca071..df4e04108 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -15,7 +15,6 @@ */ use core::mem; -use core::mem::MaybeUninit; use core::ops::{Deref, DerefMut}; use core::ptr; use core::str; @@ -604,7 +603,7 @@ pub unsafe extern "C" fn vm_init( ppool: *mut MPool, new_vm: *mut *mut Vm, ) -> bool { - match VM_MANAGER.get_mut().new_vm(vcpu_count, &mut *ppool) { + match VM_MANAGER.get_mut().new_vm(vcpu_count, &*ppool) { Some(vm) => { *new_vm = vm as *mut _; true @@ -623,7 +622,7 @@ pub unsafe extern "C" fn vm_find(id: spci_vm_id_t) -> *mut Vm { VM_MANAGER .get_mut() .vms - .get_mut(id as usize) // Ensure the VM is initialized. + .get_mut(id as usize) // Ensure the VM is initialized. .map(|vm| vm as *mut _) .unwrap_or(ptr::null_mut()) } diff --git a/inc/hf/api.h b/inc/hf/api.h index 9ba9ca534..855ad07f9 100644 --- a/inc/hf/api.h +++ b/inc/hf/api.h @@ -22,7 +22,6 @@ #include "vmapi/hf/call.h" -void api_init(struct mpool *ppool); spci_vm_id_t api_vm_get_id(const struct vcpu *current); spci_vm_count_t api_vm_get_count(void); spci_vcpu_count_t api_vcpu_get_count(spci_vm_id_t vm_id, From 7d75af69806509ca25ebbebdbfdc67ffef99d691 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 18:46:35 +0900 Subject: [PATCH 22/45] Remove vm_find. --- hfo2/src/api.rs | 73 +++++++++++++----------------------------------- hfo2/src/cpu.rs | 22 +++++++-------- hfo2/src/init.rs | 11 +++++--- hfo2/src/vm.rs | 10 ------- inc/hf/vm.h | 1 - 5 files changed, 38 insertions(+), 79 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 87fbb3cf4..967bf93ca 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -71,11 +71,8 @@ unsafe fn switch_to_primary( mut primary_ret: HfVCpuRunReturn, secondary_state: VCpuStatus, ) -> *mut VCpu { - let primary = vm_find(HF_PRIMARY_VM_ID); - let next = vm_get_vcpu( - primary, - cpu_index(current.get_inner().cpu) as spci_vcpu_index_t, - ); + let primary = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); + let next = primary.vcpus.get(cpu_index(current.get_inner().cpu)).unwrap(); // If the secondary is blocked but has a timer running, sleep until the // timer fires rather than indefinitely. @@ -95,7 +92,7 @@ unsafe fn switch_to_primary( // The use of `get_mut_unchecked()` is safe because the currently running pCPU implicitly owns // `next`. Notice that `next` is the vCPU of the primary VM that corresponds to the currently // running pCPU. - (*next) + next .inner .get_mut_unchecked() .regs @@ -104,7 +101,7 @@ unsafe fn switch_to_primary( // Mark the current vcpu as waiting. current.get_inner_mut().state = secondary_state; - next + next as *const _ as usize as *mut _ } /// Returns to the primary vm and signals that the vcpu still has work to do so. @@ -226,19 +223,14 @@ pub unsafe extern "C" fn api_vcpu_get_count( vm_id: spci_vm_id_t, current: *const VCpu, ) -> spci_vcpu_count_t { - let vm; - // Only the primary VM needs to know about vcpus for scheduling. if (*(*current).vm).id != HF_PRIMARY_VM_ID { return 0; } - vm = vm_find(vm_id); - if vm.is_null() { - return 0; - } + let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), 0); - (*vm).vcpus.len() as spci_vcpu_count_t + vm.vcpus.len() as spci_vcpu_count_t } /// This function is called by the architecture-specific context switching @@ -287,7 +279,7 @@ unsafe fn internal_interrupt_inject( /// `vcpu.execution_lock` has acquired. unsafe fn vcpu_prepare_run( current: &VCpuExecutionLocked, - vcpu: *mut VCpu, + vcpu: &VCpu, run_ret: HfVCpuRunReturn, ) -> Result { let mut vcpu_inner = (*vcpu).inner.try_lock().map_err(|_| { @@ -301,11 +293,11 @@ unsafe fn vcpu_prepare_run( run_ret })?; - if (*(*vcpu).vm).aborting.load(Ordering::Relaxed) { + if (*vcpu.vm).aborting.load(Ordering::Relaxed) { if vcpu_inner.state != VCpuStatus::Aborted { dlog!( "Aborting VM {} vCPU {}\n", - (*(*vcpu).vm).id, + (*vcpu.vm).id, vcpu_index(vcpu) ); vcpu_inner.state = VCpuStatus::Aborted; @@ -325,13 +317,13 @@ unsafe fn vcpu_prepare_run( // when it is going to be needed. This ensures there are no inter-vCPU // dependencies in the common run case meaning the sensitive context // switch performance is consistent. - VCpuStatus::BlockedMailbox if (*(*vcpu).vm).inner.lock().try_read().is_ok() => { + VCpuStatus::BlockedMailbox if (*vcpu.vm).inner.lock().try_read().is_ok() => { vcpu_inner.regs.set_retval(SpciReturn::Success as uintreg_t); } // Allow virtual interrupts to be delivered. VCpuStatus::BlockedMailbox | VCpuStatus::BlockedInterrupt - if (*vcpu).interrupts.lock().is_interrupted() => + if vcpu.interrupts.lock().is_interrupted() => { // break; } @@ -396,18 +388,13 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested VM must exist. - let vm = vm_find(vm_id); - if vm.is_null() { - return ret.into_raw(); - } + let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), ret.into_raw()); + // The requested vcpu must exist. - if vcpu_idx as usize >= (*vm).vcpus.len() { - return ret.into_raw(); - } + let vcpu = ok_or_return!(vm.vcpus.get(vcpu_idx as usize).ok_or(()), ret.into_raw()); // Update state if allowed. - let vcpu = vm_get_vcpu(vm, vcpu_idx); let mut vcpu_locked = match vcpu_prepare_run(¤t, vcpu, ret) { Ok(locked) => locked, Err(ret) => return ret.into_raw(), @@ -551,10 +538,7 @@ pub unsafe extern "C" fn api_spci_msg_send( } // Ensure the target VM exists. - let to = vm_find(from_msg_replica.target_vm_id); - if to.is_null() { - return SpciReturn::InvalidParameters; - } + let to = ok_or_return!(VM_MANAGER.get_ref().get(from_msg_replica.target_vm_id).ok_or(()), SpciReturn::InvalidParameters); // Hf needs to hold the lock on `to` before the mailbox state is checked. // The lock on `to` must be held until the information is copied to `to` Rx @@ -617,7 +601,7 @@ pub unsafe extern "C" fn api_spci_msg_send( // fields in message_buffer. The memory area message_buffer must be // exclusively owned by Hf so that TOCTOU issues do not arise. let ret = spci_msg_handle_architected_message( - &ManuallyDrop::new(VmLocked::from_raw(to)), + &ManuallyDrop::new(VmLocked::from_raw(to as *const _ as usize as *mut _)), &ManuallyDrop::new(VmLocked::from_raw(from)), architected_message_replica, &from_msg_replica, @@ -731,10 +715,7 @@ pub unsafe extern "C" fn api_mailbox_waiter_get(vm_id: spci_vm_id_t, current: *c return -1; } - let vm = vm_find(vm_id); - if vm.is_null() { - return -1; - } + let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), -1); // Check if there are outstanding notifications from given vm. let entry = (*vm).inner.lock().fetch_waiter(); @@ -834,26 +815,17 @@ pub unsafe extern "C" fn api_interrupt_inject( next: *mut *mut VCpu, ) -> i64 { let mut current = ManuallyDrop::new(VCpuExecutionLocked::from_raw(current)); - let target_vm = vm_find(target_vm_id); + let target_vm = ok_or_return!(VM_MANAGER.get_ref().get(target_vm_id).ok_or(()), -1); if intid >= HF_NUM_INTIDS { return -1; } - if target_vm.is_null() { - return -1; - } - - if target_vcpu_idx as usize >= (*target_vm).vcpus.len() { - // The requested vcpu must exist. - return -1; - } - if !is_injection_allowed(target_vm_id, current.deref()) { return -1; } - let target_vcpu = vm_get_vcpu(target_vm, target_vcpu_idx); + let target_vcpu = ok_or_return!(target_vm.vcpus.get(target_vcpu_idx as usize).ok_or(()), -1); dlog!( "Injecting IRQ {} for VM {} VCPU {} from VM {} VCPU {}\n", @@ -1007,12 +979,7 @@ fn share_memory( } // Ensure the target VM exists. - let to = unsafe { vm_find(vm_id) }; - if to.is_null() { - return Err(()); - } - - let to = unsafe { &*to }; + let to = unsafe { VM_MANAGER.get_ref() }.get(vm_id).ok_or(())?; let begin = addr; let end = ipa_add(addr, size); diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 3cf964b7c..de80054a7 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -26,6 +26,7 @@ use crate::spinlock::*; use crate::std::*; use crate::types::*; use crate::vm::*; +use crate::singleton::*; /// From inc/hf/arch/cpu.h. extern "C" { @@ -307,12 +308,16 @@ impl VCpu { interrupts: SpinLock::new(Interrupts::new()), } } + + pub fn set_cpu(&mut self, cpu: *mut Cpu) { + self.inner.get_mut().cpu = cpu; + } } /// Encapsulates a vCPU whose lock is held. #[repr(C)] pub struct VCpuExecutionLocked { - vcpu: *mut VCpu, + vcpu: *const VCpu, } impl Drop for VCpuExecutionLocked { @@ -334,12 +339,12 @@ impl Deref for VCpuExecutionLocked { impl VCpuExecutionLocked { #[inline] - pub unsafe fn from_raw(vcpu: *mut VCpu) -> Self { + pub unsafe fn from_raw(vcpu: *const VCpu) -> Self { Self { vcpu } } #[inline] - pub fn into_raw(self) -> *mut VCpu { + pub fn into_raw(self) -> *const VCpu { let ret = self.vcpu; mem::forget(self); ret @@ -455,10 +460,10 @@ pub unsafe extern "C" fn cpu_on(c: *mut Cpu, entry: ipaddr_t, arg: uintreg_t) -> *is_on = true; if !prev { - let vm = vm_find(HF_PRIMARY_VM_ID); - let vcpu = vm_get_vcpu(vm, cpu_index(c) as u16); + let vm = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); + let vcpu = vm.vcpus.get(cpu_index(c)).unwrap(); - (*vcpu).inner.lock().on(entry, arg); + vcpu.inner.lock().on(entry, arg); } prev @@ -554,11 +559,6 @@ pub unsafe extern "C" fn vcpu_get_cpu(vcpu: *mut VCpu) -> *mut Cpu { (*vcpu).inner.get_mut_unchecked().cpu } -#[no_mangle] -pub unsafe extern "C" fn vcpu_set_cpu(vcpu: *mut VCpu, cpu: *mut Cpu) { - (*vcpu).inner.get_mut_unchecked().cpu = cpu; -} - #[no_mangle] pub unsafe extern "C" fn vcpu_get_interrupts(vcpu: *mut VCpu) -> *mut Interrupts { (*vcpu).interrupts.get_mut_unchecked() diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 54467bd79..72933ab43 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -159,12 +159,15 @@ pub unsafe extern "C" fn cpu_main(c: *mut Cpu) -> *mut VCpu { panic!("mm_cpu_init failed"); } - let vcpu = vm_get_vcpu(vm_find(HF_PRIMARY_VM_ID), cpu_index(c) as spci_vcpu_index_t); - let vm = (*vcpu).vm; - vcpu_set_cpu(vcpu, c); + // TODO(HfO2): primary and vcpu are borrowed exclusively, which is safe but + // discouraged. Move this code into one_time_init(). + let primary = VM_MANAGER.get_mut().get_mut(HF_PRIMARY_VM_ID).unwrap(); + let vcpu = primary.vcpus.get_mut(cpu_index(c)).unwrap(); + let vm = vcpu.vm; + vcpu.set_cpu(c); // Reset the registers to give a clean start for the primary's vCPU. - (*vcpu) + vcpu .inner .get_mut_unchecked() .regs diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index df4e04108..c299bd6bf 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -617,16 +617,6 @@ pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { VM_MANAGER.get_ref().vms.len() as spci_vm_count_t } -#[no_mangle] -pub unsafe extern "C" fn vm_find(id: spci_vm_id_t) -> *mut Vm { - VM_MANAGER - .get_mut() - .vms - .get_mut(id as usize) // Ensure the VM is initialized. - .map(|vm| vm as *mut _) - .unwrap_or(ptr::null_mut()) -} - /// Locks the given VM and updates `locked` to hold the newly locked vm. #[no_mangle] pub unsafe extern "C" fn vm_lock(vm: *mut Vm) -> VmLocked { diff --git a/inc/hf/vm.h b/inc/hf/vm.h index c65830739..d11894fd0 100644 --- a/inc/hf/vm.h +++ b/inc/hf/vm.h @@ -94,7 +94,6 @@ struct vm_locked { bool vm_init(spci_vcpu_count_t vcpu_count, struct mpool *ppool, struct vm **new_vm); spci_vm_count_t vm_get_count(void); -struct vm *vm_find(spci_vm_id_t id); struct vm_locked vm_lock(struct vm *vm); void vm_unlock(struct vm_locked *locked); struct vcpu *vm_get_vcpu(struct vm *vm, spci_vcpu_index_t vcpu_index); From 7dbad0b665875b82583fdd72d6eb4a78332b3193 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Thu, 5 Sep 2019 23:43:24 +0900 Subject: [PATCH 23/45] Refactor CPU init sequence. --- hfo2/src/api.rs | 23 +-- hfo2/src/cpu.rs | 145 +++++++++--------- hfo2/src/init.rs | 13 +- hfo2/src/singleton.rs | 2 + inc/hf/cpu.h | 3 - .../aarch64/hypervisor/hypervisor_entry.S | 4 +- src/cpu.c | 12 +- 7 files changed, 102 insertions(+), 100 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 967bf93ca..800b34f20 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -72,7 +72,10 @@ unsafe fn switch_to_primary( secondary_state: VCpuStatus, ) -> *mut VCpu { let primary = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); - let next = primary.vcpus.get(cpu_index(current.get_inner().cpu)).unwrap(); + let next = primary + .vcpus + .get(cpu_index(&*current.get_inner().cpu)) + .unwrap(); // If the secondary is blocked but has a timer running, sleep until the // timer fires rather than indefinitely. @@ -92,8 +95,7 @@ unsafe fn switch_to_primary( // The use of `get_mut_unchecked()` is safe because the currently running pCPU implicitly owns // `next`. Notice that `next` is the vCPU of the primary VM that corresponds to the currently // running pCPU. - next - .inner + next.inner .get_mut_unchecked() .regs .set_retval(primary_ret.into_raw()); @@ -295,11 +297,7 @@ unsafe fn vcpu_prepare_run( if (*vcpu.vm).aborting.load(Ordering::Relaxed) { if vcpu_inner.state != VCpuStatus::Aborted { - dlog!( - "Aborting VM {} vCPU {}\n", - (*vcpu.vm).id, - vcpu_index(vcpu) - ); + dlog!("Aborting VM {} vCPU {}\n", (*vcpu.vm).id, vcpu_index(vcpu)); vcpu_inner.state = VCpuStatus::Aborted; } return Err(run_ret); @@ -390,7 +388,6 @@ pub unsafe extern "C" fn api_vcpu_run( // The requested VM must exist. let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), ret.into_raw()); - // The requested vcpu must exist. let vcpu = ok_or_return!(vm.vcpus.get(vcpu_idx as usize).ok_or(()), ret.into_raw()); @@ -538,7 +535,13 @@ pub unsafe extern "C" fn api_spci_msg_send( } // Ensure the target VM exists. - let to = ok_or_return!(VM_MANAGER.get_ref().get(from_msg_replica.target_vm_id).ok_or(()), SpciReturn::InvalidParameters); + let to = ok_or_return!( + VM_MANAGER + .get_ref() + .get(from_msg_replica.target_vm_id) + .ok_or(()), + SpciReturn::InvalidParameters + ); // Hf needs to hold the lock on `to` before the mailbox state is checked. // The lock on `to` must be held until the information is copied to `to` Rx diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index de80054a7..423f9f658 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -22,11 +22,13 @@ use crate::addr::*; use crate::arch::*; use crate::mm::*; use crate::page::*; +use crate::singleton::*; use crate::spinlock::*; use crate::std::*; use crate::types::*; use crate::vm::*; -use crate::singleton::*; + +use arrayvec::ArrayVec; /// From inc/hf/arch/cpu.h. extern "C" { @@ -375,6 +377,16 @@ pub struct Cpu { is_on: SpinLock, } +impl Cpu { + pub fn new(id: cpu_id_t, stack_bottom: usize) -> Self { + Self { + id, + stack_bottom: stack_bottom as *mut _, + is_on: SpinLock::new(false), + } + } +} + // unsafe impl Sync for Cpu {} /// The stack to be used by the CPUs. @@ -390,83 +402,79 @@ static mut callstacks: MaybeUninit<[[u8; STACK_SIZE]; MAX_CPUS]> = MaybeUninit:: /// Initializing static variables with pointers in Rust failed here. We left /// the initialization code of `cpus` in `src/cpu.c`. extern "C" { - #[no_mangle] - static mut cpus: MaybeUninit<[Cpu; MAX_CPUS]>; + static boot_cpu: Cpu; } -static mut CPU_COUNT: u32 = 1; - -#[no_mangle] -pub unsafe extern "C" fn cpu_init(_c: *mut Cpu) { - // TODO: Assumes that c is zeroed out already. +pub struct CpuManager { + cpus: ArrayVec<[Cpu; MAX_CPUS]>, } -#[no_mangle] -pub unsafe extern "C" fn cpu_module_init(cpu_ids: *mut cpu_id_t, count: usize) { - let mut j: u32; - let boot_cpu_id: cpu_id_t = cpus.get_ref()[0].id; - let mut found_boot_cpu: bool = false; - - arch_cpu_module_init(); - - CPU_COUNT = count as u32; - - // Initialize CPUs with the IDs from the configuration passed in. The - // CPUs after the boot CPU are initialized in reverse order. The boot - // CPU is initialized when it is found or in place of the last CPU if it - // is not found. - j = CPU_COUNT; - for i in 0..CPU_COUNT { - let c: *mut Cpu; - let id: cpu_id_t = *cpu_ids.offset(i as isize); - - if found_boot_cpu || id != boot_cpu_id { - j -= 1; - c = &mut cpus.get_mut()[j as usize]; - } else { - found_boot_cpu = true; - c = &mut cpus.get_mut()[0]; +impl CpuManager { + pub fn new( + cpu_ids: &[cpu_id_t], + boot_cpu_id: cpu_id_t, + stacks: &[[u8; STACK_SIZE]; MAX_CPUS], + ) -> Self { + arch_cpu_module_init(); + let mut cpus: ArrayVec<[Cpu; MAX_CPUS]> = ArrayVec::new(); + + // Initialize boot CPU. + let boot_stack = stacks[0].as_ptr() as usize; + cpus.push(Cpu::new(boot_cpu_id, boot_stack + STACK_SIZE)); + cpus[0].id = boot_cpu_id; + *cpus[0].is_on.get_mut() = true; + + let cpu_ids_iter = cpu_ids.iter().filter(|id| boot_cpu_id != **id); + let stacks_iter = stacks.iter().skip(1); + + for (cpu_id, stack) in cpu_ids_iter.zip(stacks_iter) { + cpus.push(Cpu::new(*cpu_id, stack.as_ptr() as usize + STACK_SIZE)); } - cpu_init(c); - (*c).id = id; - { - let callstacks_i = callstacks.get_mut()[i as usize].as_mut_ptr(); - (*c).stack_bottom = &mut *callstacks_i.add(STACK_SIZE) as *mut _ as _; + Self { cpus } + } + + pub unsafe fn cpu_index(&self, c: &Cpu) -> usize { + (c as *const Cpu).offset_from(self.cpus.as_ptr()) as usize + } + + pub unsafe fn cpu_on(&self, c: &Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { + let prev = mem::replace::(&mut c.is_on.lock(), true); + + if !prev { + let vm = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); + let vcpu = vm.vcpus.get(self.cpu_index(c)).unwrap(); - // Note: referencing callstacks.get_mut()[i as usize][STACK_SIZE] directly - // causes 'index out of bounds' error on compile time. + vcpu.inner.lock().on(entry, arg); } + + prev } - if !found_boot_cpu { - // Boot CPU was initialized but with wrong ID. - dlog!("Boot CPU's ID not found in config."); - cpus.get_mut()[0].id = boot_cpu_id; + pub fn cpu_find(&self, id: cpu_id_t) -> Option<&Cpu> { + for cpu in self.cpus.iter() { + if cpu.id == id { + return Some(cpu); + } + } + + None } } +pub fn cpu_module_init(cpu_ids: &[cpu_id_t]) -> CpuManager { + unsafe { CpuManager::new(cpu_ids, boot_cpu.id, callstacks.get_ref()) } +} + #[no_mangle] -pub unsafe extern "C" fn cpu_index(c: *mut Cpu) -> usize { - c.offset_from(cpus.get_ref().as_ptr()) as usize +pub unsafe extern "C" fn cpu_index(c: *const Cpu) -> usize { + CPU_MANAGER.get_ref().cpu_index(&*c) } /// Turns CPU on and returns the previous state. #[no_mangle] pub unsafe extern "C" fn cpu_on(c: *mut Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { - let prev: bool; - let mut is_on = (*c).is_on.lock(); - prev = *is_on; - *is_on = true; - - if !prev { - let vm = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); - let vcpu = vm.vcpus.get(cpu_index(c)).unwrap(); - - vcpu.inner.lock().on(entry, arg); - } - - prev + CPU_MANAGER.get_ref().cpu_on(&*c, entry, arg) } /// Prepares the CPU for turning itself off. @@ -478,13 +486,11 @@ pub unsafe extern "C" fn cpu_off(c: *mut Cpu) { /// Searches for a CPU based on its id. #[no_mangle] pub unsafe extern "C" fn cpu_find(id: cpu_id_t) -> *mut Cpu { - for i in 0usize..CPU_COUNT as usize { - if cpus.get_ref()[i].id == id { - return &mut cpus.get_mut()[i]; - } - } - - ptr::null_mut() + CPU_MANAGER + .get_ref() + .cpu_find(id) + .map(|cpu| cpu as *const _ as usize as *mut _) + .unwrap_or(ptr::null_mut()) } /// Locks the given vCPU and updates `locked` to hold the newly locked vCPU. @@ -515,13 +521,6 @@ pub unsafe extern "C" fn vcpu_unlock(locked: *mut VCpuExecutionLocked) { (*locked).vcpu = ptr::null_mut(); } -#[no_mangle] -pub unsafe extern "C" fn vcpu_init(vcpu: *mut VCpu, vm: *mut Vm) { - memset_s(vcpu as _, mem::size_of::(), 0, mem::size_of::()); - (*vcpu).vm = vm; - (*vcpu).inner.get_mut_unchecked().state = VCpuStatus::Off; -} - /// Initialise the registers for the given vCPU and set the state to /// VCpuStatus::Ready. The caller must hold the vCPU execution lock while /// calling this. diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 72933ab43..89b4b3273 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -15,6 +15,7 @@ */ use core::mem::{self, MaybeUninit}; +use core::slice; use crate::addr::*; use crate::api::*; @@ -65,10 +66,11 @@ unsafe fn one_time_init() { let mut hypervisor_ptable = MEMORY_MANAGER.get_ref().hypervisor_ptable.lock(); - let mut params = boot_params::get(&mut hypervisor_ptable, &mut ppool) + let params = boot_params::get(&mut hypervisor_ptable, &mut ppool) .unwrap_or_else(|| panic!("unable to retrieve boot params")); - cpu_module_init(&mut params.cpu_ids[0], params.cpu_count); + let cpu_manager = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); + CPU_MANAGER = MaybeUninit::new(cpu_manager); for i in 0..params.mem_ranges_count { dlog!( @@ -150,7 +152,7 @@ static mut INITED: bool = false; pub unsafe extern "C" fn cpu_main(c: *mut Cpu) -> *mut VCpu { // Do global one-time initialisation just once. We avoid using atomics by // only touching the variable from cpu 0. - if cpu_index(c) == 0 && !INITED { + if cpu_index(&*c) == 0 && !INITED { INITED = true; one_time_init(); } @@ -162,13 +164,12 @@ pub unsafe extern "C" fn cpu_main(c: *mut Cpu) -> *mut VCpu { // TODO(HfO2): primary and vcpu are borrowed exclusively, which is safe but // discouraged. Move this code into one_time_init(). let primary = VM_MANAGER.get_mut().get_mut(HF_PRIMARY_VM_ID).unwrap(); - let vcpu = primary.vcpus.get_mut(cpu_index(c)).unwrap(); + let vcpu = primary.vcpus.get_mut(cpu_index(&*c)).unwrap(); let vm = vcpu.vm; vcpu.set_cpu(c); // Reset the registers to give a clean start for the primary's vCPU. - vcpu - .inner + vcpu.inner .get_mut_unchecked() .regs .reset(true, &*vm, (*c).id); diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index f6f0bc4af..c40e93ff7 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -33,9 +33,11 @@ use core::mem::MaybeUninit; use crate::api::ApiManager; +use crate::cpu::CpuManager; use crate::mm::MemoryManager; use crate::vm::VmManager; pub static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); pub static mut VM_MANAGER: MaybeUninit = MaybeUninit::uninit(); pub static mut API_MANAGER: MaybeUninit = MaybeUninit::uninit(); +pub static mut CPU_MANAGER: MaybeUninit = MaybeUninit::uninit(); diff --git a/inc/hf/cpu.h b/inc/hf/cpu.h index 2f75a78c1..635f4295f 100644 --- a/inc/hf/cpu.h +++ b/inc/hf/cpu.h @@ -97,8 +97,6 @@ struct cpu { bool is_on; }; -void cpu_module_init(const cpu_id_t *cpu_ids, size_t count); - size_t cpu_index(struct cpu *c); bool cpu_on(struct cpu *c, ipaddr_t entry, uintreg_t arg); void cpu_off(struct cpu *c); @@ -107,7 +105,6 @@ struct cpu *cpu_find(cpu_id_t id); struct vcpu_execution_locked vcpu_lock(struct vcpu *vcpu); bool vcpu_try_lock(struct vcpu *vcpu, struct vcpu_execution_locked *locked); void vcpu_unlock(struct vcpu_execution_locked *locked); -void vcpu_init(struct vcpu *vcpu, struct vm *vm); void vcpu_on(struct vcpu_execution_locked vcpu, ipaddr_t entry, uintreg_t arg); spci_vcpu_index_t vcpu_index(const struct vcpu *vcpu); struct arch_regs *vcpu_get_regs(struct vcpu *vcpu); diff --git a/src/arch/aarch64/hypervisor/hypervisor_entry.S b/src/arch/aarch64/hypervisor/hypervisor_entry.S index e797074ee..55b5631c4 100644 --- a/src/arch/aarch64/hypervisor/hypervisor_entry.S +++ b/src/arch/aarch64/hypervisor/hypervisor_entry.S @@ -23,8 +23,8 @@ image_entry: bl plat_entry /* Get pointer to first cpu. */ - adrp x0, cpus - add x0, x0, :lo12:cpus + adrp x0, boot_cpu + add x0, x0, :lo12:boot_cpu /* Set the ID of this cpu from the affinity bits of mpidr. */ mrs x30, mpidr_el1 diff --git a/src/cpu.c b/src/cpu.c index fc0ab62b3..2ed923b3f 100644 --- a/src/cpu.c +++ b/src/cpu.c @@ -34,10 +34,10 @@ /* The stack to be used by the CPUs. */ extern char callstacks[MAX_CPUS][STACK_SIZE]; -/* State of all supported CPUs. The stack of the first one is initialized. */ -struct cpu cpus[MAX_CPUS] = { - { - .is_on = 1, - .stack_bottom = &callstacks[0][STACK_SIZE], - }, +/** + * A temporal variable for one-time booting sequence. The booting CPU will + * refer this. + */ +struct cpu boot_cpu = { + .stack_bottom = &callstacks[0][STACK_SIZE], }; From 0a9cb50eeb087300c4c8d946891ffb1c0d63251e Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Fri, 6 Sep 2019 13:03:19 +0900 Subject: [PATCH 24/45] Correct one-time initialization to run --- hfo2/src/cpu.rs | 8 ++++---- hfo2/src/init.rs | 11 ++++++----- inc/hf/cpu.h | 1 - 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 423f9f658..d7a3796ea 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -262,7 +262,7 @@ pub struct VCpuInner { /// ensures the scheduler can easily keep track of the vCPU state as /// transitions are indicated by the return code from the run call. pub state: VCpuStatus, - pub cpu: *mut Cpu, + pub cpu: *const Cpu, pub regs: ArchRegs, } @@ -270,7 +270,7 @@ impl VCpuInner { pub fn new() -> Self { Self { state: VCpuStatus::Off, - cpu: ptr::null_mut(), + cpu: ptr::null(), regs: ArchRegs::new(), } } @@ -311,7 +311,7 @@ impl VCpu { } } - pub fn set_cpu(&mut self, cpu: *mut Cpu) { + pub fn set_cpu(&mut self, cpu: *const Cpu) { self.inner.get_mut().cpu = cpu; } } @@ -555,7 +555,7 @@ pub unsafe extern "C" fn vcpu_get_vm(vcpu: *mut VCpu) -> *mut Vm { #[no_mangle] pub unsafe extern "C" fn vcpu_get_cpu(vcpu: *mut VCpu) -> *mut Cpu { - (*vcpu).inner.get_mut_unchecked().cpu + (*vcpu).inner.get_mut_unchecked().cpu as usize as *mut _ } #[no_mangle] diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 89b4b3273..a7d71d717 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -15,7 +15,6 @@ */ use core::mem::{self, MaybeUninit}; -use core::slice; use crate::addr::*; use crate::api::*; @@ -96,12 +95,13 @@ unsafe fn one_time_init() { let initrd = pa_addr(params.initrd_begin) as *mut _; - let mut cpio = mem::uninitialized(); + let mut cpio = MaybeUninit::uninit(); memiter_init( - &mut cpio, + cpio.get_mut(), initrd, pa_difference(params.initrd_begin, params.initrd_end), ); + let cpio = cpio.assume_init(); VM_MANAGER = MaybeUninit::new(VmManager::new()); @@ -149,12 +149,13 @@ static mut INITED: bool = false; // The entry point of CPUs when they are turned on. It is supposed to initialise // all state and return the first vCPU to run. #[no_mangle] -pub unsafe extern "C" fn cpu_main(c: *mut Cpu) -> *mut VCpu { +pub unsafe extern "C" fn cpu_main(mut c: *const Cpu) -> *mut VCpu { // Do global one-time initialisation just once. We avoid using atomics by // only touching the variable from cpu 0. - if cpu_index(&*c) == 0 && !INITED { + if !INITED { INITED = true; one_time_init(); + c = CPU_MANAGER.get_ref().cpu_find((*c).id).unwrap(); } if !mm_cpu_init() { diff --git a/inc/hf/cpu.h b/inc/hf/cpu.h index 635f4295f..b6b6b6455 100644 --- a/inc/hf/cpu.h +++ b/inc/hf/cpu.h @@ -111,7 +111,6 @@ struct arch_regs *vcpu_get_regs(struct vcpu *vcpu); const struct arch_regs *vcpu_get_regs_const(const struct vcpu *vcpu); struct vm *vcpu_get_vm(struct vcpu *vcpu); struct cpu *vcpu_get_cpu(struct vcpu *vcpu); -void vcpu_set_cpu(struct vcpu *vcpu, struct cpu *cpu); struct interrupts *vcpu_get_interrupts(struct vcpu *vcpu); bool vcpu_is_off(struct vcpu_execution_locked vcpu); bool vcpu_secondary_reset_and_start(struct vcpu *vcpu, ipaddr_t entry, From d6b350b6872be9e4652cd68b972a64c959794ab5 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Fri, 6 Sep 2019 14:20:22 +0900 Subject: [PATCH 25/45] Make safe wrapper functions to read singletons. --- hfo2/src/api.rs | 25 ++++++++---------- hfo2/src/cpu.rs | 14 +++++----- hfo2/src/init.rs | 42 +++++++++++++++++++----------- hfo2/src/load.rs | 19 ++++++++------ hfo2/src/mm.rs | 18 ++++--------- hfo2/src/singleton.rs | 59 ++++++++++++++++++++++++++++++++++++++++--- hfo2/src/vm.rs | 15 ++++++++--- 7 files changed, 128 insertions(+), 64 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 800b34f20..88c687eb5 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -71,7 +71,7 @@ unsafe fn switch_to_primary( mut primary_ret: HfVCpuRunReturn, secondary_state: VCpuStatus, ) -> *mut VCpu { - let primary = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); + let primary = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); let next = primary .vcpus .get(cpu_index(&*current.get_inner().cpu)) @@ -230,7 +230,7 @@ pub unsafe extern "C" fn api_vcpu_get_count( return 0; } - let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), 0); + let vm = ok_or_return!(vm_manager().get(vm_id).ok_or(()), 0); vm.vcpus.len() as spci_vcpu_count_t } @@ -386,7 +386,7 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested VM must exist. - let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), ret.into_raw()); + let vm = ok_or_return!(vm_manager().get(vm_id).ok_or(()), ret.into_raw()); // The requested vcpu must exist. let vcpu = ok_or_return!(vm.vcpus.get(vcpu_idx as usize).ok_or(()), ret.into_raw()); @@ -479,7 +479,7 @@ pub unsafe extern "C" fn api_vm_configure( // to keep a single unlock point. let mut vm_inner = (*vm).inner.lock(); if vm_inner - .configure(send, recv, &API_MANAGER.get_ref().ppool) + .configure(send, recv, &api_manager().ppool) .is_err() { return -1; @@ -536,10 +536,7 @@ pub unsafe extern "C" fn api_spci_msg_send( // Ensure the target VM exists. let to = ok_or_return!( - VM_MANAGER - .get_ref() - .get(from_msg_replica.target_vm_id) - .ok_or(()), + vm_manager().get(from_msg_replica.target_vm_id).ok_or(()), SpciReturn::InvalidParameters ); @@ -718,7 +715,7 @@ pub unsafe extern "C" fn api_mailbox_waiter_get(vm_id: spci_vm_id_t, current: *c return -1; } - let vm = ok_or_return!(VM_MANAGER.get_ref().get(vm_id).ok_or(()), -1); + let vm = ok_or_return!(vm_manager().get(vm_id).ok_or(()), -1); // Check if there are outstanding notifications from given vm. let entry = (*vm).inner.lock().fetch_waiter(); @@ -818,7 +815,7 @@ pub unsafe extern "C" fn api_interrupt_inject( next: *mut *mut VCpu, ) -> i64 { let mut current = ManuallyDrop::new(VCpuExecutionLocked::from_raw(current)); - let target_vm = ok_or_return!(VM_MANAGER.get_ref().get(target_vm_id).ok_or(()), -1); + let target_vm = ok_or_return!(vm_manager().get(target_vm_id).ok_or(()), -1); if intid >= HF_NUM_INTIDS { return -1; @@ -844,7 +841,7 @@ pub unsafe extern "C" fn api_interrupt_inject( /// Clears a region of physical memory by overwriting it with zeros. The data is /// flushed from the cache so the memory has been cleared across the system. fn clear_memory(begin: paddr_t, end: paddr_t, ppool: &MPool) -> Result<(), ()> { - let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.hypervisor_ptable.lock(); + let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); let size = pa_difference(begin, end); let region = pa_addr(begin); @@ -901,7 +898,7 @@ pub fn spci_share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool: MPool = MPool::new_with_fallback(unsafe { &API_MANAGER.get_ref().ppool }); + let local_page_pool: MPool = MPool::new_with_fallback(&api_manager().ppool); // Obtain the single contiguous set of pages from the memory_region. // TODO: Add support for multiple constituent regions. @@ -982,7 +979,7 @@ fn share_memory( } // Ensure the target VM exists. - let to = unsafe { VM_MANAGER.get_ref() }.get(vm_id).ok_or(())?; + let to = vm_manager().get(vm_id).ok_or(())?; let begin = addr; let end = ipa_add(addr, size); @@ -1007,7 +1004,7 @@ fn share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool = MPool::new_with_fallback(unsafe { &API_MANAGER.get_ref().ppool }); + let local_page_pool = MPool::new_with_fallback(&api_manager().ppool); let (mut from_inner, mut to_inner) = SpinLock::lock_both(&(*from).inner, &(*to).inner); diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index d7a3796ea..9f0833d53 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -394,8 +394,8 @@ impl Cpu { #[no_mangle] static mut callstacks: MaybeUninit<[[u8; STACK_SIZE]; MAX_CPUS]> = MaybeUninit::uninit(); -/// State of all supported CPUs. The stack of the first one is initialized. -/// Kernel loader writes booted CPU ID on `cpus.id` and initializes the CPU +/// A record for boot CPU. Its field `stack_bottom` is initialized. +/// Hafnium loader writes booted CPU ID on `cpus.id` and initializes the CPU /// stack by the address in `cpus.stack_bottom`. /// (See src/arch/aarch64/hypervisor/plat_entry.S and cpu_entry.S.) /// @@ -406,6 +406,7 @@ extern "C" { } pub struct CpuManager { + /// State of all supported CPUs. cpus: ArrayVec<[Cpu; MAX_CPUS]>, } @@ -442,7 +443,7 @@ impl CpuManager { let prev = mem::replace::(&mut c.is_on.lock(), true); if !prev { - let vm = VM_MANAGER.get_ref().get(HF_PRIMARY_VM_ID).unwrap(); + let vm = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); let vcpu = vm.vcpus.get(self.cpu_index(c)).unwrap(); vcpu.inner.lock().on(entry, arg); @@ -468,13 +469,13 @@ pub fn cpu_module_init(cpu_ids: &[cpu_id_t]) -> CpuManager { #[no_mangle] pub unsafe extern "C" fn cpu_index(c: *const Cpu) -> usize { - CPU_MANAGER.get_ref().cpu_index(&*c) + cpu_manager().cpu_index(&*c) } /// Turns CPU on and returns the previous state. #[no_mangle] pub unsafe extern "C" fn cpu_on(c: *mut Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { - CPU_MANAGER.get_ref().cpu_on(&*c, entry, arg) + cpu_manager().cpu_on(&*c, entry, arg) } /// Prepares the CPU for turning itself off. @@ -486,8 +487,7 @@ pub unsafe extern "C" fn cpu_off(c: *mut Cpu) { /// Searches for a CPU based on its id. #[no_mangle] pub unsafe extern "C" fn cpu_find(id: cpu_id_t) -> *mut Cpu { - CPU_MANAGER - .get_ref() + cpu_manager() .cpu_find(id) .map(|cpu| cpu as *const _ as usize as *mut _) .unwrap_or(ptr::null_mut()) diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index a7d71d717..78bb5d026 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -56,20 +56,20 @@ unsafe fn one_time_init() { mem::size_of_val(PTABLE_BUF.get_ref()), ); - let memory_manager = MemoryManager::new(&ppool).expect("mm_init failed"); - MEMORY_MANAGER = MaybeUninit::new(memory_manager); + let mm = MemoryManager::new(&ppool).expect("mm_init failed"); + memory_manager_init(mm); // Enable locks now that mm is initialised. dlog_enable_lock(); mpool_enable_locks(); - let mut hypervisor_ptable = MEMORY_MANAGER.get_ref().hypervisor_ptable.lock(); + let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); let params = boot_params::get(&mut hypervisor_ptable, &mut ppool) .unwrap_or_else(|| panic!("unable to retrieve boot params")); let cpu_manager = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); - CPU_MANAGER = MaybeUninit::new(cpu_manager); + cpu_manager_init(cpu_manager); for i in 0..params.mem_ranges_count { dlog!( @@ -103,11 +103,17 @@ unsafe fn one_time_init() { ); let cpio = cpio.assume_init(); - VM_MANAGER = MaybeUninit::new(VmManager::new()); + let mut vm_manager = VmManager::new(); // Load all VMs. - let primary_initrd = load_primary(&mut hypervisor_ptable, &cpio, params.kernel_arg, &ppool) - .unwrap_or_else(|_| panic!("unable to load primary VM")); + let primary_initrd = load_primary( + &mut vm_manager, + &mut hypervisor_ptable, + &cpio, + params.kernel_arg, + &ppool, + ) + .unwrap_or_else(|_| panic!("unable to load primary VM")); // load_secondary will add regions assigned to the secondary VMs from // mem_ranges to reserved_ranges. @@ -117,6 +123,7 @@ unsafe fn one_time_init() { ); if load_secondary( + &mut vm_manager, &mut hypervisor_ptable, &cpio, ¶ms, @@ -128,6 +135,8 @@ unsafe fn one_time_init() { panic!("unable to load secondary VMs"); } + vm_manager_init(vm_manager); + // Prepare to run by updating bootparams as seen by primary VM. if boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool).is_err() { panic!("plat_update_boot_params failed"); @@ -136,7 +145,7 @@ unsafe fn one_time_init() { hypervisor_ptable.defrag(&ppool); // Initialise the API page pool. ppool will be empty from now on. - API_MANAGER = MaybeUninit::new(ApiManager::new(ppool)); + api_manager_init(ApiManager::new(ppool)); // Enable TLB invalidation for VM page table updates. mm_vm_enable_invalidation(); @@ -155,22 +164,25 @@ pub unsafe extern "C" fn cpu_main(mut c: *const Cpu) -> *mut VCpu { if !INITED { INITED = true; one_time_init(); - c = CPU_MANAGER.get_ref().cpu_find((*c).id).unwrap(); + c = cpu_manager().cpu_find((*c).id).unwrap(); } if !mm_cpu_init() { panic!("mm_cpu_init failed"); } - // TODO(HfO2): primary and vcpu are borrowed exclusively, which is safe but - // discouraged. Move this code into one_time_init(). - let primary = VM_MANAGER.get_mut().get_mut(HF_PRIMARY_VM_ID).unwrap(); - let vcpu = primary.vcpus.get_mut(cpu_index(&*c)).unwrap(); + let primary = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); + let vcpu = primary.vcpus.get(cpu_index(&*c)).unwrap(); let vm = vcpu.vm; - vcpu.set_cpu(c); + + // TODO(HfO2): vcpu needs to be borrowed exclusively, which is safe but + // discouraged. Move this code into one_time_init(). + let vcpu = vcpu as *const _ as usize as *mut VCpu; + (*vcpu).set_cpu(c); // Reset the registers to give a clean start for the primary's vCPU. - vcpu.inner + (*vcpu) + .inner .get_mut_unchecked() .regs .reset(true, &*vm, (*c).id); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 8f4fb09aa..d11e9d63a 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -30,6 +30,7 @@ use crate::singleton::*; use crate::std::*; use crate::types::*; use crate::utils::*; +use crate::vm::*; use arrayvec::ArrayVec; @@ -65,6 +66,7 @@ unsafe fn copy_to_unmapped( /// Loads the primary VM. pub unsafe fn load_primary( + vm_manager: &mut VmManager, hypervisor_ptable: &mut PageTable, cpio: &MemIter, kernel_arg: uintreg_t, @@ -101,8 +103,7 @@ pub unsafe fn load_primary( let initrd = initrd.assume_init(); - let vm = VM_MANAGER - .get_mut() + let vm = vm_manager .new_vm(MAX_CPUS as spci_vcpu_count_t, ppool) .ok_or_else(|| { dlog!("Unable to initialise primary vm\n"); @@ -199,6 +200,7 @@ fn update_reserved_ranges( /// Loads all secondary VMs into the memory ranges from the given params. /// Memory reserved for the VMs is added to the `reserved_ranges` of `update`. pub unsafe fn load_secondary( + vm_manager: &mut VmManager, hypervisor_ptable: &mut PageTable, cpio: &MemIter, params: &BootParams, @@ -215,7 +217,7 @@ pub unsafe fn load_secondary( mem_ranges_available.clone_from_slice(¶ms.mem_ranges); mem_ranges_available.truncate(params.mem_ranges_count); - let primary = VM_MANAGER.get_mut().get_mut(HF_PRIMARY_VM_ID).ok_or(())?; + let primary = vm_manager.get_mut(HF_PRIMARY_VM_ID).ok_or(())?; let mut it = mem::uninitialized(); if !cpio_find_file(cpio, "vms.txt\0".as_ptr(), &mut it) { @@ -279,17 +281,18 @@ pub unsafe fn load_secondary( continue; } - // TODO(HfO2): This should be rejected by borrowck, IMO. (by defining - // primary, VM_MANAGER is borrowed exclusively before.) Maybe unsafe - // static mut confused borrowck? - let vm = match VM_MANAGER.get_mut().new_vm(cpu as spci_vcpu_count_t, ppool) { - Some(vm) => vm, + let vm_id = match vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool) { + Some(vm) => vm.id, None => { dlog!("Unable to initialise VM\n"); continue; } }; + // We have to borrow primary again here, due to conservative Rust borrow + // checker. + let (primary, vm) = vm_manager.get_mut_with_primary(vm_id).unwrap(); + let secondary_entry = ipa_from_pa(secondary_mem_begin); // Grant the VM access to the memory. diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index 5d7523f1b..c258f45ff 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -264,10 +264,7 @@ impl Stage for Stage2 { } fn invalidate_tlb(begin: usize, end: usize) { - if unsafe { MEMORY_MANAGER.get_ref() } - .stage2_invalidate - .load(Ordering::Relaxed) - { + if memory_manager().stage2_invalidate.load(Ordering::Relaxed) { unsafe { arch_mm_invalidate_stage2_range(ipa_init(begin), ipa_init(end)); } @@ -1104,8 +1101,7 @@ impl MemoryManager { /// This function should not be invoked concurrently with other memory management functions. #[no_mangle] pub unsafe extern "C" fn mm_vm_enable_invalidation() { - MEMORY_MANAGER - .get_ref() + memory_manager() .stage2_invalidate .store(true, Ordering::Relaxed); } @@ -1242,18 +1238,14 @@ pub unsafe extern "C" fn mm_unmap( #[no_mangle] pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { let mm = ok_or_return!(MemoryManager::new(&*mpool).ok_or(()), false); - MEMORY_MANAGER = MaybeUninit::new(mm); + memory_manager_init(mm); true } #[no_mangle] pub unsafe extern "C" fn mm_cpu_init() -> bool { - let raw_ptable = MEMORY_MANAGER - .get_ref() - .hypervisor_ptable - .get_mut_unchecked() - .root; + let raw_ptable = memory_manager().hypervisor_ptable.get_mut_unchecked().root; arch_mm_init(raw_ptable, false) } @@ -1265,7 +1257,7 @@ pub unsafe extern "C" fn mm_defrag(mut stage1_locked: mm_stage1_locked, mpool: * #[no_mangle] pub unsafe extern "C" fn mm_lock_stage1() -> mm_stage1_locked { - let ptable = &MEMORY_MANAGER.get_ref().hypervisor_ptable; + let ptable = &memory_manager().hypervisor_ptable; ptable.lock().into() } diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index c40e93ff7..2ab9612e8 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -29,6 +29,22 @@ //! //! Therefore, I do not use a safe wrapper for initialization such as //! `std::sync::Once` and `lazy_static`. +//! +//! # Safety +//! +//! Hafnium has two states representing its overall life. +//! - Initializing. +//! - Exception handling. +//! +//! Hafnium starts from initializing but eventually moves into exception +//! handling state. Thus I treat exception handling state as if it were +//! Hafnium's _usual_ mode; that's why `{memory, vm, api, cpu}_manager` +//! functions are not denoted by `unsafe`. On the other hand, initializers must +//! not be called in the handling state, so that they are `unsafe`. +//! +//! Setting singleton getters as safe causes one problem, that calling one of +//! those functions in initalizing sequence is dangerous if the singleton were +//! not yet initialized. Watch your code! use core::mem::MaybeUninit; @@ -37,7 +53,42 @@ use crate::cpu::CpuManager; use crate::mm::MemoryManager; use crate::vm::VmManager; -pub static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); -pub static mut VM_MANAGER: MaybeUninit = MaybeUninit::uninit(); -pub static mut API_MANAGER: MaybeUninit = MaybeUninit::uninit(); -pub static mut CPU_MANAGER: MaybeUninit = MaybeUninit::uninit(); +// Singletons. +static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); +static mut VM_MANAGER: MaybeUninit = MaybeUninit::uninit(); +static mut API_MANAGER: MaybeUninit = MaybeUninit::uninit(); +static mut CPU_MANAGER: MaybeUninit = MaybeUninit::uninit(); + +// Singleton initializers. +pub unsafe fn memory_manager_init(mm: MemoryManager) { + MEMORY_MANAGER = MaybeUninit::new(mm); +} + +pub unsafe fn vm_manager_init(vmm: VmManager) { + VM_MANAGER = MaybeUninit::new(vmm); +} + +pub unsafe fn api_manager_init(apim: ApiManager) { + API_MANAGER = MaybeUninit::new(apim); +} + +pub unsafe fn cpu_manager_init(cpum: CpuManager) { + CPU_MANAGER = MaybeUninit::new(cpum); +} + +// Global functions to acquire read-only references to singletons. +pub fn memory_manager() -> &'static MemoryManager { + unsafe { MEMORY_MANAGER.get_ref() } +} + +pub fn vm_manager() -> &'static VmManager { + unsafe { VM_MANAGER.get_ref() } +} + +pub fn api_manager() -> &'static ApiManager { + unsafe { API_MANAGER.get_ref() } +} + +pub fn cpu_manager() -> &'static CpuManager { + unsafe { CPU_MANAGER.get_ref() } +} diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index c299bd6bf..e8e480300 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -156,7 +156,7 @@ impl Mailbox { ) -> Result<(), ()> { // TODO(HfO2): Acquring the singleton here is not recommended. Get the // hypervisor ptable from callee (API module.) - let mut hypervisor_ptable = unsafe { MEMORY_MANAGER.get_ref() }.hypervisor_ptable.lock(); + let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); let mut ptable = guard(hypervisor_ptable.deref_mut(), |_| ()); // Map the send page as read-only in the hypervisor address space. @@ -595,15 +595,24 @@ impl VmManager { pub fn get_mut(&mut self, id: spci_vm_id_t) -> Option<&mut Vm> { self.vms.get_mut(id as usize) } + + pub fn get_mut_with_primary(&mut self, id: spci_vm_id_t) -> Option<(&mut Vm, &mut Vm)> { + let (primary, rest) = self.vms.split_first_mut()?; + let vm = rest.get_mut((id - 1) as usize)?; + + Some((primary, vm)) + } } +/// This function is only used by unit test (fdt/find_memory_ranges.) #[no_mangle] pub unsafe extern "C" fn vm_init( vcpu_count: spci_vcpu_count_t, ppool: *mut MPool, new_vm: *mut *mut Vm, ) -> bool { - match VM_MANAGER.get_mut().new_vm(vcpu_count, &*ppool) { + let vmm = vm_manager() as *const _ as usize as *mut VmManager; + match (*vmm).new_vm(vcpu_count, &*ppool) { Some(vm) => { *new_vm = vm as *mut _; true @@ -614,7 +623,7 @@ pub unsafe extern "C" fn vm_init( #[no_mangle] pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { - VM_MANAGER.get_ref().vms.len() as spci_vm_count_t + vm_manager().vms.len() as spci_vm_count_t } /// Locks the given VM and updates `locked` to hold the newly locked vm. From 7dac8a454667481a25a3078361432aee187bf3d1 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Fri, 6 Sep 2019 16:24:38 +0900 Subject: [PATCH 26/45] Fix Vm to know its address correctly while initialized --- hfo2/src/init.rs | 8 +++----- hfo2/src/load.rs | 1 - hfo2/src/singleton.rs | 7 +++++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 78bb5d026..a48064125 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -103,11 +103,11 @@ unsafe fn one_time_init() { ); let cpio = cpio.assume_init(); - let mut vm_manager = VmManager::new(); + vm_manager_init(VmManager::new()); // Load all VMs. let primary_initrd = load_primary( - &mut vm_manager, + vm_manager_mut(), &mut hypervisor_ptable, &cpio, params.kernel_arg, @@ -123,7 +123,7 @@ unsafe fn one_time_init() { ); if load_secondary( - &mut vm_manager, + vm_manager_mut(), &mut hypervisor_ptable, &cpio, ¶ms, @@ -135,8 +135,6 @@ unsafe fn one_time_init() { panic!("unable to load secondary VMs"); } - vm_manager_init(vm_manager); - // Prepare to run by updating bootparams as seen by primary VM. if boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool).is_err() { panic!("plat_update_boot_params failed"); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index d11e9d63a..157b7a11d 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -217,7 +217,6 @@ pub unsafe fn load_secondary( mem_ranges_available.clone_from_slice(¶ms.mem_ranges); mem_ranges_available.truncate(params.mem_ranges_count); - let primary = vm_manager.get_mut(HF_PRIMARY_VM_ID).ok_or(())?; let mut it = mem::uninitialized(); if !cpio_find_file(cpio, "vms.txt\0".as_ptr(), &mut it) { diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index 2ab9612e8..d9982dde0 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -92,3 +92,10 @@ pub fn api_manager() -> &'static ApiManager { pub fn cpu_manager() -> &'static CpuManager { unsafe { CPU_MANAGER.get_ref() } } + +/// A temporal function to load VM correctly. Today VM need to know its address +/// to initialize itself, thus we cannot simply 'move' the initialized VmManager +/// into VM_MANAGER. TODO(HfO2): Fix it, and remove this function. +pub unsafe fn vm_manager_mut() -> &'static mut VmManager { + VM_MANAGER.get_mut() +} From 54c3e18c78277a91eecd2191fd54666544640d9d Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Fri, 6 Sep 2019 16:30:13 +0900 Subject: [PATCH 27/45] Remove unused imports --- hfo2/src/load.rs | 1 - hfo2/src/mm.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 157b7a11d..ceeca79e2 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -26,7 +26,6 @@ use crate::memiter::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; -use crate::singleton::*; use crate::std::*; use crate::types::*; use crate::utils::*; diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index c258f45ff..9458e6a59 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -27,7 +27,7 @@ use core::cmp; use core::marker::PhantomData; -use core::mem::{self, MaybeUninit}; +use core::mem; use core::ops::*; use core::ptr; use core::slice; From eebc8535af627d316be25d99a1e63682c816fc4b Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sun, 8 Sep 2019 02:56:33 +0900 Subject: [PATCH 28/45] Resolve simple issues on the review of #45 --- hfo2/src/abi.rs | 2 +- hfo2/src/boot_params.rs | 2 ++ hfo2/src/cpu.rs | 21 ++++++++++++--------- hfo2/src/fdt.rs | 4 +++- hfo2/src/init.rs | 2 +- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/hfo2/src/abi.rs b/hfo2/src/abi.rs index c3a7dbb5a..8f803a2c7 100644 --- a/hfo2/src/abi.rs +++ b/hfo2/src/abi.rs @@ -122,7 +122,7 @@ mod test { /// Encode a preempted response without leaking. #[test] - fn abi_hf_vcpu_run_return_encode_preemptd() { + fn abi_hf_vcpu_run_return_encode_preempted() { let res = HfVCpuRunReturn::Preempted; assert_eq!(res.into_raw(), 0); } diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs index e3547be14..399a4d36a 100644 --- a/hfo2/src/boot_params.rs +++ b/hfo2/src/boot_params.rs @@ -86,6 +86,7 @@ extern "C" { ) -> bool; } +/// Reads platform-specific boot parameters. pub fn get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> Option { unsafe { let mut p: MaybeUninit = MaybeUninit::uninit(); @@ -98,6 +99,7 @@ pub fn get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> } } +/// Updates boot parameters for primary VM to read. pub fn update( ptable: &mut SpinLockGuard>, p: &mut BootParamsUpdate, diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 9f0833d53..1b66b2e0c 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -107,6 +107,7 @@ impl Interrupts { enabled_and_pending_count: 0, } } + pub fn id_to_index(intid: intid_t) -> Result<(usize, u32), ()> { if intid >= HF_NUM_INTIDS { return Err(()); @@ -425,26 +426,28 @@ impl CpuManager { cpus[0].id = boot_cpu_id; *cpus[0].is_on.get_mut() = true; - let cpu_ids_iter = cpu_ids.iter().filter(|id| boot_cpu_id != **id); - let stacks_iter = stacks.iter().skip(1); + let cpu_ids = cpu_ids.iter().filter(|id| boot_cpu_id != **id); + let stacks = stacks.iter().skip(1); - for (cpu_id, stack) in cpu_ids_iter.zip(stacks_iter) { + for (cpu_id, stack) in cpu_ids.zip(stacks) { cpus.push(Cpu::new(*cpu_id, stack.as_ptr() as usize + STACK_SIZE)); } Self { cpus } } - pub unsafe fn cpu_index(&self, c: &Cpu) -> usize { + pub unsafe fn index_of(&self, c: &Cpu) -> usize { (c as *const Cpu).offset_from(self.cpus.as_ptr()) as usize } pub unsafe fn cpu_on(&self, c: &Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { - let prev = mem::replace::(&mut c.is_on.lock(), true); + let mut is_on = c.is_on.lock(); + let prev = *is_on; + *is_on = true; if !prev { let vm = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); - let vcpu = vm.vcpus.get(self.cpu_index(c)).unwrap(); + let vcpu = vm.vcpus.get(self.index_of(c)).unwrap(); vcpu.inner.lock().on(entry, arg); } @@ -452,7 +455,7 @@ impl CpuManager { prev } - pub fn cpu_find(&self, id: cpu_id_t) -> Option<&Cpu> { + pub fn lookup(&self, id: cpu_id_t) -> Option<&Cpu> { for cpu in self.cpus.iter() { if cpu.id == id { return Some(cpu); @@ -469,7 +472,7 @@ pub fn cpu_module_init(cpu_ids: &[cpu_id_t]) -> CpuManager { #[no_mangle] pub unsafe extern "C" fn cpu_index(c: *const Cpu) -> usize { - cpu_manager().cpu_index(&*c) + cpu_manager().index_of(&*c) } /// Turns CPU on and returns the previous state. @@ -488,7 +491,7 @@ pub unsafe extern "C" fn cpu_off(c: *mut Cpu) { #[no_mangle] pub unsafe extern "C" fn cpu_find(id: cpu_id_t) -> *mut Cpu { cpu_manager() - .cpu_find(id) + .lookup(id) .map(|cpu| cpu as *const _ as usize as *mut _) .unwrap_or(ptr::null_mut()) } diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 8785b08ce..d95bd3761 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -30,6 +30,7 @@ extern "C" { } #[derive(Clone)] +#[repr(C)] pub struct FdtNode { hdr: *const FdtHeader, begin: *const u8, @@ -90,6 +91,7 @@ struct FdtTokenizer { const FDT_VERSION: u32 = 17; const FDT_MAGIC: u32 = 0xd00d_feed; +const FDT_TOKEN_ALIGNMENT: usize = mem::size_of::(); impl FdtTokenizer { fn new(strs: *const u8, begin: *const u8, end: *const u8) -> Self { @@ -101,7 +103,7 @@ impl FdtTokenizer { } fn align(&mut self) { - self.cur = round_up(self.cur as usize, 4) as _; + self.cur = round_up(self.cur as usize, FDT_TOKEN_ALIGNMENT) as _; } unsafe fn u32(&mut self) -> Option { diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index a48064125..d303302e8 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -162,7 +162,7 @@ pub unsafe extern "C" fn cpu_main(mut c: *const Cpu) -> *mut VCpu { if !INITED { INITED = true; one_time_init(); - c = cpu_manager().cpu_find((*c).id).unwrap(); + c = cpu_manager().lookup((*c).id).unwrap(); } if !mm_cpu_init() { From 9e52fae65cf0f341ca6cc527b86e2fc0974248d0 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sun, 8 Sep 2019 03:07:55 +0900 Subject: [PATCH 29/45] More idiomatic init of Cpu --- hfo2/src/cpu.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 1b66b2e0c..f65dc3d7b 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -379,11 +379,11 @@ pub struct Cpu { } impl Cpu { - pub fn new(id: cpu_id_t, stack_bottom: usize) -> Self { + pub fn new(id: cpu_id_t, stack_bottom: usize, is_on: bool) -> Self { Self { id, stack_bottom: stack_bottom as *mut _, - is_on: SpinLock::new(false), + is_on: SpinLock::new(is_on), } } } @@ -422,15 +422,17 @@ impl CpuManager { // Initialize boot CPU. let boot_stack = stacks[0].as_ptr() as usize; - cpus.push(Cpu::new(boot_cpu_id, boot_stack + STACK_SIZE)); - cpus[0].id = boot_cpu_id; - *cpus[0].is_on.get_mut() = true; + cpus.push(Cpu::new(boot_cpu_id, boot_stack + STACK_SIZE, true)); let cpu_ids = cpu_ids.iter().filter(|id| boot_cpu_id != **id); let stacks = stacks.iter().skip(1); for (cpu_id, stack) in cpu_ids.zip(stacks) { - cpus.push(Cpu::new(*cpu_id, stack.as_ptr() as usize + STACK_SIZE)); + cpus.push(Cpu::new( + *cpu_id, + stack.as_ptr() as usize + STACK_SIZE, + false, + )); } Self { cpus } From cfdb6227a98e4a13f048ef451dfa1ef9bdbdba45 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Sun, 8 Sep 2019 19:24:24 +0900 Subject: [PATCH 30/45] Resolve simple issues on the review of #45 (cont'd) --- hfo2/src/api.rs | 2 +- hfo2/src/fdt.rs | 60 ++++++++++++++++++------------------------- hfo2/src/init.rs | 35 ++++++++----------------- hfo2/src/load.rs | 55 +++++++++++++++++---------------------- hfo2/src/types.rs | 2 +- hfo2/src/utils.rs | 10 ++++++++ hfo2/src/vm.rs | 11 ++------ kokoro/ubuntu/test.sh | 2 +- 8 files changed, 73 insertions(+), 104 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 88c687eb5..1c57aa3bb 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -230,7 +230,7 @@ pub unsafe extern "C" fn api_vcpu_get_count( return 0; } - let vm = ok_or_return!(vm_manager().get(vm_id).ok_or(()), 0); + let vm = unwrap_or!(vm_manager().get(vm_id), return 0); vm.vcpus.len() as spci_vcpu_count_t } diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index d95bd3761..a20d5e4eb 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -20,7 +20,6 @@ use core::ptr; use core::slice; use core::str; -use crate::std::*; use crate::utils::*; use scopeguard::guard; @@ -312,10 +311,7 @@ impl FdtNode { impl FdtHeader { pub unsafe fn dump(&self) { unsafe fn asciz_to_utf8(ptr: *const u8) -> &'static str { - let mut len = 0; - while *ptr.add(len) != 0u8 { - len += 1; - } + let len = (0..).find(|i| *ptr.add(*i) != 0).unwrap(); let bytes = slice::from_raw_parts(ptr, len); str::from_utf8_unchecked(bytes) } @@ -370,25 +366,24 @@ impl FdtHeader { "fdt: off_mem_rsvmap={}\n", u32::from_be(self.off_mem_rsvmap) ); - { - let e = self as *const _ as usize + u32::from_be(self.off_mem_rsvmap) as usize; - let mut entry = e as *const FdtReserveEntry; - while (*entry).address != 0 || (*entry).size != 0 { - dlog!( - "Entry: {:p} (0x{:x} bytes)\n", - u64::from_be((*entry).address) as *const u8, - u64::from_be((*entry).size) - ); - entry = entry.add(1); - } + let mut entry = (self as *const _ as usize + u32::from_be(self.off_mem_rsvmap) as usize) + as *const FdtReserveEntry; + + while (*entry).address != 0 || (*entry).size != 0 { + dlog!( + "Entry: {:p} (0x{:x} bytes)\n", + u64::from_be((*entry).address) as *const u8, + u64::from_be((*entry).size) + ); + entry = entry.add(1); } } pub unsafe fn add_mem_reservation(&mut self, addr: u64, len: u64) { // TODO: Clean this up. - let begin = (self as *const _ as usize as *const u8) - .add(u32::from_be(self.off_mem_rsvmap) as usize); + let begin = + (self as *const _ as usize as *mut u8).add(u32::from_be(self.off_mem_rsvmap) as usize); let e = begin as *mut FdtReserveEntry; let old_size = u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap); @@ -399,12 +394,12 @@ impl FdtHeader { self.off_dt_strings = (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); - memmove_s( - begin.add(mem::size_of::()) as usize as *mut _, - old_size as usize, - begin as usize as *const _, + ptr::copy_nonoverlapping( + begin, + begin.add(mem::size_of::()), old_size as usize, ); + (*e).address = addr.to_be(); (*e).size = len.to_be(); } @@ -420,7 +415,7 @@ pub extern "C" fn fdt_header_size() -> usize { } #[no_mangle] -pub unsafe extern "C" fn fdt_total_size(hdr: *mut FdtHeader) -> u32 { +pub unsafe extern "C" fn fdt_total_size(hdr: *const FdtHeader) -> u32 { (*hdr).total_size() } @@ -444,10 +439,9 @@ pub unsafe extern "C" fn fdt_find_child(node: *mut FdtNode, child: *const u8) -> #[no_mangle] pub unsafe extern "C" fn fdt_first_child(node: *mut FdtNode, child_name: *mut *const u8) -> bool { - (*node).first_child().map_or(false, |name| { - ptr::write(child_name, name); - true - }) + let name = unwrap_or!((*node).first_child(), return false); + ptr::write(child_name, name); + true } #[no_mangle] @@ -468,14 +462,10 @@ pub unsafe extern "C" fn fdt_read_property( buf: *mut *const u8, size: *mut u32, ) -> bool { - match (*node).read_property(name) { - Ok((prop_buf, prop_size)) => { - *buf = prop_buf; - *size = prop_size; - true - } - Err(_) => false, - } + let (prop_buf, prop_size) = ok_or_return!((*node).read_property(name), false); + *buf = prop_buf; + *size = prop_size; + true } #[no_mangle] diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index d303302e8..5eb6d60d4 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -66,7 +66,7 @@ unsafe fn one_time_init() { let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); let params = boot_params::get(&mut hypervisor_ptable, &mut ppool) - .unwrap_or_else(|| panic!("unable to retrieve boot params")); + .expect("unable to retrieve boot params"); let cpu_manager = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); cpu_manager_init(cpu_manager); @@ -86,22 +86,15 @@ unsafe fn one_time_init() { ); // Map initrd in, and initialise cpio parser. - if hypervisor_ptable + hypervisor_ptable .identity_map(params.initrd_begin, params.initrd_end, Mode::R, &ppool) - .is_err() - { - panic!("unable to map initrd in"); - } + .expect("unable to map initrd in"); let initrd = pa_addr(params.initrd_begin) as *mut _; - - let mut cpio = MaybeUninit::uninit(); - memiter_init( - cpio.get_mut(), + let cpio = MemIter::from_raw( initrd, pa_difference(params.initrd_begin, params.initrd_end), ); - let cpio = cpio.assume_init(); vm_manager_init(VmManager::new()); @@ -113,7 +106,7 @@ unsafe fn one_time_init() { params.kernel_arg, &ppool, ) - .unwrap_or_else(|_| panic!("unable to load primary VM")); + .expect("unable to load primary VM"); // load_secondary will add regions assigned to the secondary VMs from // mem_ranges to reserved_ranges. @@ -122,7 +115,7 @@ unsafe fn one_time_init() { pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)), ); - if load_secondary( + load_secondary( vm_manager_mut(), &mut hypervisor_ptable, &cpio, @@ -130,15 +123,11 @@ unsafe fn one_time_init() { &mut update, &mut ppool, ) - .is_err() - { - panic!("unable to load secondary VMs"); - } + .expect("unable to load secondary VMs"); // Prepare to run by updating bootparams as seen by primary VM. - if boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool).is_err() { - panic!("plat_update_boot_params failed"); - } + boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool) + .expect("plat_update_boot_params failed"); hypervisor_ptable.defrag(&ppool); @@ -179,11 +168,7 @@ pub unsafe extern "C" fn cpu_main(mut c: *const Cpu) -> *mut VCpu { (*vcpu).set_cpu(c); // Reset the registers to give a clean start for the primary's vCPU. - (*vcpu) - .inner - .get_mut_unchecked() - .regs - .reset(true, &*vm, (*c).id); + (*vcpu).inner.get_mut().regs.reset(true, &*vm, (*c).id); vcpu } diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index ceeca79e2..d0d633a93 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -14,7 +14,7 @@ * limitations under the License. */ -use core::mem::{self, MaybeUninit}; +use core::mem; use crate::addr::*; use crate::arch::*; @@ -71,18 +71,18 @@ pub unsafe fn load_primary( kernel_arg: uintreg_t, ppool: &MPool, ) -> Result { - let mut it = mem::uninitialized(); let primary_begin = layout_primary_begin(); - if !cpio_find_file(cpio, "vmlinuz\0".as_ptr(), &mut it) { + let it = unwrap_or!(find_file(&mut cpio.clone(), "vmlinuz\0".as_ptr()), { dlog!("Unable to find vmlinuz\n"); return Err(()); - } + }); dlog!( "Copying primary to {:p}\n", pa_addr(primary_begin) as *const u8 ); + if !copy_to_unmapped( hypervisor_ptable, primary_begin, @@ -94,13 +94,10 @@ pub unsafe fn load_primary( return Err(()); } - let mut initrd = MaybeUninit::uninit(); - if !cpio_find_file(cpio, "initrd.img\0".as_ptr(), initrd.get_mut()) { + let initrd = unwrap_or!(find_file(&mut cpio.clone(), "initrd.img\0".as_ptr()), { dlog!("Unable to find initrd.img\n"); return Err(()); - } - - let initrd = initrd.assume_init(); + }); let vm = vm_manager .new_vm(MAX_CPUS as spci_vcpu_count_t, ppool) @@ -216,26 +213,23 @@ pub unsafe fn load_secondary( mem_ranges_available.clone_from_slice(¶ms.mem_ranges); mem_ranges_available.truncate(params.mem_ranges_count); - let mut it = mem::uninitialized(); - - if !cpio_find_file(cpio, "vms.txt\0".as_ptr(), &mut it) { + let mut it = unwrap_or!(find_file(&mut cpio.clone(), "vms.txt\0".as_ptr()), { dlog!("vms.txt is missing\n"); return Ok(()); - } + }); // Round the last addresses down to the page size. for mem_range in mem_ranges_available.iter_mut() { mem_range.end = pa_init(round_down(pa_addr(mem_range.end), PAGE_SIZE)); } - let mut mem = mem::uninitialized(); - let mut cpu = mem::uninitialized(); - let mut name = mem::uninitialized(); + loop { + // Note(HfO2): There is `while let (Some(x), Some(y)) = (...) {}` but it + // is not short-circuiting. + let mut mem = unwrap_or!(it.parse_uint(), break); + let cpu = unwrap_or!(it.parse_uint(), break); + let name = unwrap_or!(it.parse_str(), break); - while memiter_parse_uint(&mut it, &mut mem) - && memiter_parse_uint(&mut it, &mut cpu) - && memiter_parse_str(&mut it, &mut name) - { dlog!("Loading "); let mut p = name.next; while p != name.limit { @@ -244,12 +238,10 @@ pub unsafe fn load_secondary( } dlog!("\n"); - let mut kernel = mem::uninitialized(); - - if !cpio_find_file_memiter(cpio, &name, &mut kernel) { + let kernel = unwrap_or!(find_file_memiter(&mut cpio.clone(), &name), { dlog!("Unable to load kernel\n"); continue; - } + }); // Round up to page size. mem = (mem + PAGE_SIZE as u64 - 1) & !(PAGE_SIZE as u64 - 1); @@ -279,20 +271,14 @@ pub unsafe fn load_secondary( continue; } - let vm_id = match vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool) { - Some(vm) => vm.id, + let vm = match vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool) { + Some(vm) => vm, None => { dlog!("Unable to initialise VM\n"); continue; } }; - // We have to borrow primary again here, due to conservative Rust borrow - // checker. - let (primary, vm) = vm_manager.get_mut_with_primary(vm_id).unwrap(); - - let secondary_entry = ipa_from_pa(secondary_mem_begin); - // Grant the VM access to the memory. if vm .inner @@ -310,6 +296,9 @@ pub unsafe fn load_secondary( continue; } + let vm_id = vm.id; + let primary = vm_manager.get_mut(HF_PRIMARY_VM_ID).unwrap(); + // Deny the primary VM access to this memory. if primary .inner @@ -328,6 +317,8 @@ pub unsafe fn load_secondary( pa_addr(secondary_mem_begin) ); + let vm = vm_manager.get_mut(vm_id).unwrap(); + let secondary_entry = ipa_from_pa(secondary_mem_begin); vcpu_secondary_reset_and_start( &mut vm.vcpus[0], secondary_entry, diff --git a/hfo2/src/types.rs b/hfo2/src/types.rs index 2acb08bc1..d7db3dea0 100644 --- a/hfo2/src/types.rs +++ b/hfo2/src/types.rs @@ -57,7 +57,7 @@ pub const HF_INVALID_INTID: intid_t = 0xffff_ffff; /// The virtual interrupt ID used for the virtual timer. pub const HF_VIRTUAL_TIMER_INTID: intid_t = 3; -// These constants are originally from build scripts. (See +// TODO(HfO2): These constants are originally from build scripts. (See // //project/reference/BUILD.gn.) pub const HEAP_PAGES: usize = 60; diff --git a/hfo2/src/utils.rs b/hfo2/src/utils.rs index 0a0c67294..8114d5b70 100644 --- a/hfo2/src/utils.rs +++ b/hfo2/src/utils.rs @@ -26,6 +26,16 @@ macro_rules! ok_or_return { }}; } +#[macro_export] +macro_rules! unwrap_or { + ($e:expr, $err:expr) => {{ + match $e { + Some(r) => r, + None => $err, + } + }}; +} + pub fn spin_loop() -> ! { loop { spin_loop_hint(); diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index e8e480300..5c9baa1d4 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -595,13 +595,6 @@ impl VmManager { pub fn get_mut(&mut self, id: spci_vm_id_t) -> Option<&mut Vm> { self.vms.get_mut(id as usize) } - - pub fn get_mut_with_primary(&mut self, id: spci_vm_id_t) -> Option<(&mut Vm, &mut Vm)> { - let (primary, rest) = self.vms.split_first_mut()?; - let vm = rest.get_mut((id - 1) as usize)?; - - Some((primary, vm)) - } } /// This function is only used by unit test (fdt/find_memory_ranges.) @@ -623,7 +616,7 @@ pub unsafe extern "C" fn vm_init( #[no_mangle] pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { - vm_manager().vms.len() as spci_vm_count_t + vm_manager().vms.len() as _ } /// Locks the given VM and updates `locked` to hold the newly locked vm. @@ -666,5 +659,5 @@ pub unsafe extern "C" fn vm_get_arch(vm: *mut Vm) -> *mut ArchVm { #[no_mangle] pub unsafe extern "C" fn vm_get_vcpu_count(vm: *const Vm) -> spci_vcpu_count_t { - (*vm).vcpus.len() as spci_vcpu_count_t + (*vm).vcpus.len() as _ } diff --git a/kokoro/ubuntu/test.sh b/kokoro/ubuntu/test.sh index 9c22fdb46..492452a2a 100755 --- a/kokoro/ubuntu/test.sh +++ b/kokoro/ubuntu/test.sh @@ -61,7 +61,7 @@ $TIMEOUT 30s $OUT/host_fake_clang/unit_tests \ --gtest_output="xml:$OUT/kokoro_log/unit_tests/sponge_log.xml" \ | tee $OUT/kokoro_log/unit_tests/sponge_log.log -cd hfo2 && cargo test && cd .. +cargo test --manifest-path=hfo2/Cargo.toml $HFTEST arch_test $HFTEST hafnium --initrd test/vmapi/gicv3/gicv3_test From 7fc3b7ff0b757a05dbdde27f078aa563ef95af5c Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Mon, 9 Sep 2019 12:58:08 +0900 Subject: [PATCH 31/45] Fix a bug introduced from cfdb6227a98e4a13f048ef451dfa1ef9bdbdba45 --- hfo2/src/fdt.rs | 31 +++++++++++++------------------ hfo2/src/fdt_handler.rs | 24 +++++++++--------------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index a20d5e4eb..c73db60e8 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -317,13 +317,10 @@ impl FdtHeader { } // Traverse the whole thing. - let node = match FdtNode::new_root(self) { - Some(node) => node, - None => { - dlog!("FDT failed validation.\n"); - return; - } - }; + let node = unwrap_or!(FdtNode::new_root(self), { + dlog!("FDT failed validation.\n"); + return; + }); let mut t = FdtTokenizer::new(node.strs, node.begin, node.end); let mut depth = 0; @@ -385,7 +382,7 @@ impl FdtHeader { let begin = (self as *const _ as usize as *mut u8).add(u32::from_be(self.off_mem_rsvmap) as usize); let e = begin as *mut FdtReserveEntry; - let old_size = u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap); + let old_size = (u32::from_be(self.totalsize) - u32::from_be(self.off_mem_rsvmap)) as usize; self.totalsize = (u32::from_be(self.totalsize) + mem::size_of::() as u32).to_be(); @@ -394,10 +391,10 @@ impl FdtHeader { self.off_dt_strings = (u32::from_be(self.off_dt_strings) + mem::size_of::() as u32).to_be(); - ptr::copy_nonoverlapping( + ptr::copy( begin, begin.add(mem::size_of::()), - old_size as usize, + old_size, ); (*e).address = addr.to_be(); @@ -426,10 +423,9 @@ pub unsafe extern "C" fn fdt_dump(hdr: *mut FdtHeader) { #[no_mangle] pub unsafe extern "C" fn fdt_root_node(node: *mut FdtNode, hdr: *const FdtHeader) -> bool { - FdtNode::new_root(&*hdr).map_or(false, |n| { - ptr::write(node, n); - true - }) + let n = unwrap_or!(FdtNode::new_root(&*hdr), return false); + ptr::write(node, n); + true } #[no_mangle] @@ -449,10 +445,9 @@ pub unsafe extern "C" fn fdt_next_sibling( node: *mut FdtNode, sibling_name: *mut *const u8, ) -> bool { - (*node).next_sibling().map_or(false, |name| { - ptr::write(sibling_name, name); - true - }) + let name = unwrap_or!((*node).next_sibling(), return false); + ptr::write(sibling_name, name); + true } #[no_mangle] diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index 886b8c5f5..a72b4ee9a 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -72,21 +72,15 @@ impl FdtNode { return false; } - let initrd_begin = match self.read_number("linux,initrd-start\0".as_ptr()) { - Ok(initrd_begin) => initrd_begin, - Err(_) => { - dlog!("Unable to read linux,initrd-start\n"); - return false; - } - }; - - let initrd_end = match self.read_number("linux,initrd-end\0".as_ptr()) { - Ok(initrd_end) => initrd_end, - Err(_) => { - dlog!("Unable to read linux,initrd-end\n"); - return false; - } - }; + let initrd_begin = ok_or_return!(self.read_number("linux,initrd-start\0".as_ptr()), { + dlog!("Unable to read linux,initrd-start\n"); + false + }); + + let initrd_end = ok_or_return!(self.read_number("linux,initrd-end\0".as_ptr()), { + dlog!("Unable to read linux,initrd-end\n"); + false + }); *begin = pa_init(initrd_begin as usize); *end = pa_init(initrd_end as usize); From 5f7a90b4c37a6c8ba605b91af32a0617ee880d86 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Mon, 9 Sep 2019 20:12:47 +0900 Subject: [PATCH 32/45] Resolve simple issues on the review of #45 (cont'd) --- hfo2/src/arch/fake.rs | 3 +++ hfo2/src/boot_params.rs | 21 ++++++++++----------- hfo2/src/cpu.rs | 2 +- hfo2/src/fdt.rs | 1 + hfo2/src/init.rs | 14 +++++--------- hfo2/src/load.rs | 30 ++++++++++++++---------------- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/hfo2/src/arch/fake.rs b/hfo2/src/arch/fake.rs index 6c30c60a0..4079f748a 100644 --- a/hfo2/src/arch/fake.rs +++ b/hfo2/src/arch/fake.rs @@ -55,6 +55,9 @@ pub fn arch_cpu_module_init() { // Do nothing. } +// TODO(HfO2): Following functions are empty, since linker complains if the +// implementations of functions are missing even they're never called in the +// unit tests. Make a custom target and remove those (#46.) #[no_mangle] pub extern "C" fn arch_one_time_init() { unreachable!(); diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs index 399a4d36a..46faeadc3 100644 --- a/hfo2/src/boot_params.rs +++ b/hfo2/src/boot_params.rs @@ -25,20 +25,16 @@ use crate::types::*; pub const MAX_MEM_RANGES: usize = 20; -#[derive(Clone)] +#[derive(Clone, Copy)] #[repr(C)] pub struct MemRange { pub begin: paddr_t, pub end: paddr_t, } -/// TODO(HfO2): Is it more natural for `paddr_t` to have 0 as default value? -impl Default for MemRange { - fn default() -> Self { - Self { - begin: pa_init(0), - end: pa_init(0), - } +impl MemRange { + pub fn new(begin: paddr_t, end: paddr_t) -> Self { + Self { begin, end } } } @@ -64,7 +60,7 @@ pub struct BootParamsUpdate { impl BootParamsUpdate { pub fn new(initrd_begin: paddr_t, initrd_end: paddr_t) -> Self { Self { - reserved_ranges: Default::default(), + reserved_ranges: [MemRange::new(pa_init(0), pa_init(0)); MAX_MEM_RANGES], reserved_ranges_count: 0, initrd_begin, initrd_end, @@ -72,6 +68,9 @@ impl BootParamsUpdate { } } +/// TODO(HfO2): `plat.c`, containing those functions are not ported into Rust. +/// It's because functions in `plat.c` are denoted by `#pragma weak` which is +/// not supported in Rust yet. (#47.) extern "C" { fn plat_get_boot_params( stage1_locked: mm_stage1_locked, @@ -87,7 +86,7 @@ extern "C" { } /// Reads platform-specific boot parameters. -pub fn get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> Option { +pub fn boot_params_get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> Option { unsafe { let mut p: MaybeUninit = MaybeUninit::uninit(); @@ -100,7 +99,7 @@ pub fn get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> } /// Updates boot parameters for primary VM to read. -pub fn update( +pub fn boot_params_update( ptable: &mut SpinLockGuard>, p: &mut BootParamsUpdate, ppool: &mut MPool, diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index f65dc3d7b..4bd1f1206 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -559,7 +559,7 @@ pub unsafe extern "C" fn vcpu_get_vm(vcpu: *mut VCpu) -> *mut Vm { } #[no_mangle] -pub unsafe extern "C" fn vcpu_get_cpu(vcpu: *mut VCpu) -> *mut Cpu { +pub unsafe extern "C" fn vcpu_get_cpu(vcpu: *const VCpu) -> *mut Cpu { (*vcpu).inner.get_mut_unchecked().cpu as usize as *mut _ } diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index c73db60e8..45cfbe9db 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -24,6 +24,7 @@ use crate::utils::*; use scopeguard::guard; +/// TODO(HfO2): port this function into `std.rs` (#48.) extern "C" { fn strcmp(a: *const u8, b: *const u8) -> i64; } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 5eb6d60d4..8cf76bf93 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -14,11 +14,11 @@ * limitations under the License. */ -use core::mem::{self, MaybeUninit}; +use core::mem::MaybeUninit; use crate::addr::*; use crate::api::*; -use crate::boot_params::{self, BootParamsUpdate}; +use crate::boot_params::*; use crate::cpu::*; use crate::load::*; use crate::memiter::*; @@ -50,11 +50,7 @@ unsafe fn one_time_init() { arch_one_time_init(); let mut ppool = MPool::new(); - mpool_add_chunk( - &mut ppool, - PTABLE_BUF.get_mut().as_mut_ptr() as usize as *mut _, - mem::size_of_val(PTABLE_BUF.get_ref()), - ); + ppool.free_pages(Pages::from_raw(PTABLE_BUF.get_mut().as_mut_ptr(), HEAP_PAGES)); let mm = MemoryManager::new(&ppool).expect("mm_init failed"); memory_manager_init(mm); @@ -65,7 +61,7 @@ unsafe fn one_time_init() { let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); - let params = boot_params::get(&mut hypervisor_ptable, &mut ppool) + let params = boot_params_get(&mut hypervisor_ptable, &mut ppool) .expect("unable to retrieve boot params"); let cpu_manager = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); @@ -126,7 +122,7 @@ unsafe fn one_time_init() { .expect("unable to load secondary VMs"); // Prepare to run by updating bootparams as seen by primary VM. - boot_params::update(&mut hypervisor_ptable, &mut update, &mut ppool) + boot_params_update(&mut hypervisor_ptable, &mut update, &mut ppool) .expect("plat_update_boot_params failed"); hypervisor_ptable.defrag(&ppool); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index d0d633a93..733fc5e3a 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -271,6 +271,20 @@ pub unsafe fn load_secondary( continue; } + let primary = vm_manager.get_mut(HF_PRIMARY_VM_ID).unwrap(); + + // Deny the primary VM access to this memory. + if primary + .inner + .get_mut() + .ptable + .unmap(secondary_mem_begin, secondary_mem_end, ppool) + .is_err() + { + dlog!("Unable to unmap secondary VM from primary VM\n"); + return Err(()); + } + let vm = match vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool) { Some(vm) => vm, None => { @@ -296,28 +310,12 @@ pub unsafe fn load_secondary( continue; } - let vm_id = vm.id; - let primary = vm_manager.get_mut(HF_PRIMARY_VM_ID).unwrap(); - - // Deny the primary VM access to this memory. - if primary - .inner - .get_mut() - .ptable - .unmap(secondary_mem_begin, secondary_mem_end, ppool) - .is_err() - { - dlog!("Unable to unmap secondary VM from primary VM\n"); - return Err(()); - } - dlog!( "Loaded with {} vcpus, entry at 0x{:x}\n", cpu, pa_addr(secondary_mem_begin) ); - let vm = vm_manager.get_mut(vm_id).unwrap(); let secondary_entry = ipa_from_pa(secondary_mem_begin); vcpu_secondary_reset_and_start( &mut vm.vcpus[0], From 1f8d79a29ee73544fdd321a4b982a9831362c520 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Tue, 10 Sep 2019 00:37:27 +0900 Subject: [PATCH 33/45] Call one_time_init directly in hypervisor_entry, so remove INITED. --- hfo2/src/cpu.rs | 5 +++ hfo2/src/init.rs | 21 ++++-------- .../aarch64/hypervisor/hypervisor_entry.S | 33 ++++++++++++++++++- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 4bd1f1206..80de078f2 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -466,6 +466,11 @@ impl CpuManager { None } + + /// Temporal impl (WIP) + pub fn boot_cpu(&self) -> *mut Cpu { + self.cpus.get(0).unwrap() as &Cpu as *const _ as usize as *mut _ + } } pub fn cpu_module_init(cpu_ids: &[cpu_id_t]) -> CpuManager { diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 8cf76bf93..c3cbcc4f2 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -41,7 +41,8 @@ extern "C" { static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); /// Performs one-time initialisation of the hypervisor. -unsafe fn one_time_init() { +#[no_mangle] +unsafe extern "C" fn one_time_init() -> *mut Cpu { // Make sure the console is initialised before calling dlog. plat_console_init(); @@ -64,8 +65,8 @@ unsafe fn one_time_init() { let params = boot_params_get(&mut hypervisor_ptable, &mut ppool) .expect("unable to retrieve boot params"); - let cpu_manager = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); - cpu_manager_init(cpu_manager); + let cpum = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); + cpu_manager_init(cpum); for i in 0..params.mem_ranges_count { dlog!( @@ -134,22 +135,14 @@ unsafe fn one_time_init() { mm_vm_enable_invalidation(); dlog!("Hafnium initialisation completed\n"); -} -static mut INITED: bool = false; + cpu_manager().boot_cpu() +} // The entry point of CPUs when they are turned on. It is supposed to initialise // all state and return the first vCPU to run. #[no_mangle] -pub unsafe extern "C" fn cpu_main(mut c: *const Cpu) -> *mut VCpu { - // Do global one-time initialisation just once. We avoid using atomics by - // only touching the variable from cpu 0. - if !INITED { - INITED = true; - one_time_init(); - c = cpu_manager().lookup((*c).id).unwrap(); - } - +pub unsafe extern "C" fn cpu_main(c: *const Cpu) -> *mut VCpu { if !mm_cpu_init() { panic!("mm_cpu_init failed"); } diff --git a/src/arch/aarch64/hypervisor/hypervisor_entry.S b/src/arch/aarch64/hypervisor/hypervisor_entry.S index 55b5631c4..e528e3acc 100644 --- a/src/arch/aarch64/hypervisor/hypervisor_entry.S +++ b/src/arch/aarch64/hypervisor/hypervisor_entry.S @@ -33,4 +33,35 @@ image_entry: orr x30, x29, x30 str x30, [x0, CPU_ID] - b cpu_entry + /** + * Basically do the same thing on cpu_entry, but call one_time_init + * before cpu_main. + */ + + /* Disable interrupts. */ + msr DAIFSet, #0xf + + /* Use SPx (instead of SP0). */ + msr spsel, #1 + + /* Prepare the stack. */ + ldr x30, [x0, #CPU_STACK_BOTTOM] + mov sp, x30 + + /* Configure exception handlers. */ + adrp x30, vector_table_el2 + add x30, x30, :lo12:vector_table_el2 + msr vbar_el2, x30 + + /* Initialize the hypervisor. It returns a correct pointer to boot cpu. */ + bl one_time_init + + /* Call into C code, x0 holds the cpu pointer. */ + bl cpu_main + + /* Run the vcpu returned by cpu_main. */ + bl vcpu_restore_all_and_run + + /* Loop forever waiting for interrupts. */ +0: wfi + b 0b From cf675b4c14c299714775e926ef6ecb5790faaf6a Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Tue, 10 Sep 2019 00:49:47 +0900 Subject: [PATCH 34/45] Remove cpu_module_init and move related statics to init.rs --- hfo2/src/cpu.rs | 24 +----------------------- hfo2/src/init.rs | 16 +++++++++++++++- src/cpu.c | 2 +- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 80de078f2..77fd722f3 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -62,7 +62,7 @@ extern "C" { fn arch_regs_set_retval(r: *mut ArchRegs, v: uintreg_t); } -const STACK_SIZE: usize = PAGE_SIZE; +pub const STACK_SIZE: usize = PAGE_SIZE; /// The number of bits in each element of the interrupt bitfields. pub const INTERRUPT_REGISTER_BITS: usize = 32; @@ -388,24 +388,6 @@ impl Cpu { } } -// unsafe impl Sync for Cpu {} - -/// The stack to be used by the CPUs. -/// TODO: alignas(2 * sizeof(uintreg_t)) -#[no_mangle] -static mut callstacks: MaybeUninit<[[u8; STACK_SIZE]; MAX_CPUS]> = MaybeUninit::uninit(); - -/// A record for boot CPU. Its field `stack_bottom` is initialized. -/// Hafnium loader writes booted CPU ID on `cpus.id` and initializes the CPU -/// stack by the address in `cpus.stack_bottom`. -/// (See src/arch/aarch64/hypervisor/plat_entry.S and cpu_entry.S.) -/// -/// Initializing static variables with pointers in Rust failed here. We left -/// the initialization code of `cpus` in `src/cpu.c`. -extern "C" { - static boot_cpu: Cpu; -} - pub struct CpuManager { /// State of all supported CPUs. cpus: ArrayVec<[Cpu; MAX_CPUS]>, @@ -473,10 +455,6 @@ impl CpuManager { } } -pub fn cpu_module_init(cpu_ids: &[cpu_id_t]) -> CpuManager { - unsafe { CpuManager::new(cpu_ids, boot_cpu.id, callstacks.get_ref()) } -} - #[no_mangle] pub unsafe extern "C" fn cpu_index(c: *const Cpu) -> usize { cpu_manager().index_of(&*c) diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index c3cbcc4f2..2cc10cbc3 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -40,6 +40,20 @@ extern "C" { /// but it was not aligned to PAGE_SIZE. static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); +extern "C" { + /// The stack to be used by the CPUs. + static callstacks: [[u8; STACK_SIZE]; MAX_CPUS]; + + /// A record for boot CPU. Its field `stack_bottom` is initialized. + /// Hafnium loader writes booted CPU ID on `cpus.id` and initializes the CPU + /// stack by the address in `cpus.stack_bottom`. + /// (See src/arch/aarch64/hypervisor/plat_entry.S and cpu_entry.S.) + /// + /// Initializing static variables with pointers in Rust failed here. We left + /// the initialization code of `cpus` in `src/cpu.c`. + static boot_cpu: Cpu; +} + /// Performs one-time initialisation of the hypervisor. #[no_mangle] unsafe extern "C" fn one_time_init() -> *mut Cpu { @@ -65,7 +79,7 @@ unsafe extern "C" fn one_time_init() -> *mut Cpu { let params = boot_params_get(&mut hypervisor_ptable, &mut ppool) .expect("unable to retrieve boot params"); - let cpum = cpu_module_init(¶ms.cpu_ids[..params.cpu_count]); + let cpum = CpuManager::new(¶ms.cpu_ids[..params.cpu_count], boot_cpu.id, &callstacks); cpu_manager_init(cpum); for i in 0..params.mem_ranges_count { diff --git a/src/cpu.c b/src/cpu.c index 2ed923b3f..b832f9798 100644 --- a/src/cpu.c +++ b/src/cpu.c @@ -32,7 +32,7 @@ #define STACK_SIZE PAGE_SIZE /* The stack to be used by the CPUs. */ -extern char callstacks[MAX_CPUS][STACK_SIZE]; +alignas(2 * sizeof(uintreg_t)) char callstacks[MAX_CPUS][STACK_SIZE]; /** * A temporal variable for one-time booting sequence. The booting CPU will From 9d61512dcd982564e8fa9d8aaec1df8c3f0227c0 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Tue, 10 Sep 2019 22:34:16 +0900 Subject: [PATCH 35/45] Fix stack overflow during Vm initialization. --- hfo2/src/vm.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index 5c9baa1d4..2b4432687 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -14,7 +14,7 @@ * limitations under the License. */ -use core::mem; +use core::mem::{self, MaybeUninit}; use core::ops::{Deref, DerefMut}; use core::ptr; use core::str; @@ -485,14 +485,26 @@ impl Vm { ppool: &MPool, ) -> Result<(), ()> { self.id = id; - self.vcpus = ArrayVec::new(); + // self.vcpus = ArrayVec::new(); + // TODO(HfO2): Using former one will allocate large temporal stack to + // store a ArrayVec<[VCpu; MAX_CPUS]>. Maybe MIR-only RLIBs or denoting + // ArrayVec::new as inline will solve this problem. + // (See https://github.com/rust-lang/rust/issues/38913) + unsafe { + self.vcpus = MaybeUninit::uninit().assume_init(); + self.vcpus.set_len(0); + } self.aborting = AtomicBool::new(false); unsafe { let self_ptr = self as *mut _; self.inner.get_mut().init(self_ptr, ppool)?; for _ in 0..vcpu_count { - self.vcpus.push(VCpu::new(self_ptr)); + // self.vcpus.push(VCpu::new(self_ptr)); + // TODO(HfO2): Using former one will allocate large temporal + // stack to store a VCpu. Maybe MIR-only RLIBs or denoting + // ArrayVec::push as inline will solve this problem. + self.vcpus.push_unchecked(VCpu::new(self_ptr)); } } Ok(()) From 7a371f5d7514851dc32ca2964854720bca746df9 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 11:10:40 +0900 Subject: [PATCH 36/45] Call arch_cpu_module_init() in init.rs --- hfo2/src/cpu.rs | 1 - hfo2/src/init.rs | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 77fd722f3..e306f38dd 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -399,7 +399,6 @@ impl CpuManager { boot_cpu_id: cpu_id_t, stacks: &[[u8; STACK_SIZE]; MAX_CPUS], ) -> Self { - arch_cpu_module_init(); let mut cpus: ArrayVec<[Cpu; MAX_CPUS]> = ArrayVec::new(); // Initialize boot CPU. diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 2cc10cbc3..451bee653 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -17,6 +17,7 @@ use core::mem::MaybeUninit; use crate::addr::*; +use crate::arch::*; use crate::api::*; use crate::boot_params::*; use crate::cpu::*; @@ -79,6 +80,8 @@ unsafe extern "C" fn one_time_init() -> *mut Cpu { let params = boot_params_get(&mut hypervisor_ptable, &mut ppool) .expect("unable to retrieve boot params"); + arch_cpu_module_init(); + let cpum = CpuManager::new(¶ms.cpu_ids[..params.cpu_count], boot_cpu.id, &callstacks); cpu_manager_init(cpum); From 950dd1d2b640403eecd68649260d9f898c1d819f Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 12:53:48 +0900 Subject: [PATCH 37/45] Revert 1f8d79a, but call one_time_init in cpu_entry. --- hfo2/src/boot_params.rs | 7 +++- hfo2/src/init.rs | 38 +++++++++++++------ src/arch/aarch64/hypervisor/cpu_entry.S | 3 ++ .../aarch64/hypervisor/hypervisor_entry.S | 25 +----------- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/hfo2/src/boot_params.rs b/hfo2/src/boot_params.rs index 46faeadc3..b969a3877 100644 --- a/hfo2/src/boot_params.rs +++ b/hfo2/src/boot_params.rs @@ -69,7 +69,7 @@ impl BootParamsUpdate { } /// TODO(HfO2): `plat.c`, containing those functions are not ported into Rust. -/// It's because functions in `plat.c` are denoted by `#pragma weak` which is +/// It's because functions in `plat.c` are denoted by `#pragma weak` which is /// not supported in Rust yet. (#47.) extern "C" { fn plat_get_boot_params( @@ -86,7 +86,10 @@ extern "C" { } /// Reads platform-specific boot parameters. -pub fn boot_params_get(ptable: &mut SpinLockGuard>, ppool: &mut MPool) -> Option { +pub fn boot_params_get( + ptable: &mut SpinLockGuard>, + ppool: &mut MPool, +) -> Option { unsafe { let mut p: MaybeUninit = MaybeUninit::uninit(); diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 451bee653..391373cbf 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -17,8 +17,8 @@ use core::mem::MaybeUninit; use crate::addr::*; -use crate::arch::*; use crate::api::*; +use crate::arch::*; use crate::boot_params::*; use crate::cpu::*; use crate::load::*; @@ -34,14 +34,7 @@ extern "C" { fn plat_console_init(); fn arch_one_time_init(); fn dlog_enable_lock(); -} - -/// Note(HfO2): this variable was originally of type -/// MaybeUninit<[u8; mem::size_of::() * HEAP_PAGES]>, -/// but it was not aligned to PAGE_SIZE. -static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); -extern "C" { /// The stack to be used by the CPUs. static callstacks: [[u8; STACK_SIZE]; MAX_CPUS]; @@ -55,9 +48,20 @@ extern "C" { static boot_cpu: Cpu; } +/// Note(HfO2): this variable was originally of type +/// MaybeUninit<[u8; mem::size_of::() * HEAP_PAGES]>, +/// but it was not aligned to PAGE_SIZE. +static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); + +static mut INITED: bool = false; + /// Performs one-time initialisation of the hypervisor. #[no_mangle] -unsafe extern "C" fn one_time_init() -> *mut Cpu { +unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { + if INITED { + return c; + } + // Make sure the console is initialised before calling dlog. plat_console_init(); @@ -66,7 +70,10 @@ unsafe extern "C" fn one_time_init() -> *mut Cpu { arch_one_time_init(); let mut ppool = MPool::new(); - ppool.free_pages(Pages::from_raw(PTABLE_BUF.get_mut().as_mut_ptr(), HEAP_PAGES)); + ppool.free_pages(Pages::from_raw( + PTABLE_BUF.get_mut().as_mut_ptr(), + HEAP_PAGES, + )); let mm = MemoryManager::new(&ppool).expect("mm_init failed"); memory_manager_init(mm); @@ -82,7 +89,11 @@ unsafe extern "C" fn one_time_init() -> *mut Cpu { arch_cpu_module_init(); - let cpum = CpuManager::new(¶ms.cpu_ids[..params.cpu_count], boot_cpu.id, &callstacks); + let cpum = CpuManager::new( + ¶ms.cpu_ids[..params.cpu_count], + boot_cpu.id, + &callstacks, + ); cpu_manager_init(cpum); for i in 0..params.mem_ranges_count { @@ -152,8 +163,13 @@ unsafe extern "C" fn one_time_init() -> *mut Cpu { mm_vm_enable_invalidation(); dlog!("Hafnium initialisation completed\n"); + INITED = true; cpu_manager().boot_cpu() + + // From now on, other pCPUs are on in order to run multiple vCPUs. Thus + // you may safely make readonly references to the Hafnium singleton, but + // may not modify the singleton without proper locking. } // The entry point of CPUs when they are turned on. It is supposed to initialise diff --git a/src/arch/aarch64/hypervisor/cpu_entry.S b/src/arch/aarch64/hypervisor/cpu_entry.S index 301a1a10d..316fe0435 100644 --- a/src/arch/aarch64/hypervisor/cpu_entry.S +++ b/src/arch/aarch64/hypervisor/cpu_entry.S @@ -34,6 +34,9 @@ cpu_entry: add x30, x30, :lo12:vector_table_el2 msr vbar_el2, x30 + /* Initialize Hafnium. */ + bl one_time_init + /* Call into C code, x0 holds the cpu pointer. */ bl cpu_main diff --git a/src/arch/aarch64/hypervisor/hypervisor_entry.S b/src/arch/aarch64/hypervisor/hypervisor_entry.S index e528e3acc..8cdb0647d 100644 --- a/src/arch/aarch64/hypervisor/hypervisor_entry.S +++ b/src/arch/aarch64/hypervisor/hypervisor_entry.S @@ -33,31 +33,8 @@ image_entry: orr x30, x29, x30 str x30, [x0, CPU_ID] - /** - * Basically do the same thing on cpu_entry, but call one_time_init - * before cpu_main. - */ - - /* Disable interrupts. */ - msr DAIFSet, #0xf - - /* Use SPx (instead of SP0). */ - msr spsel, #1 - - /* Prepare the stack. */ - ldr x30, [x0, #CPU_STACK_BOTTOM] - mov sp, x30 - - /* Configure exception handlers. */ - adrp x30, vector_table_el2 - add x30, x30, :lo12:vector_table_el2 - msr vbar_el2, x30 - - /* Initialize the hypervisor. It returns a correct pointer to boot cpu. */ - bl one_time_init - /* Call into C code, x0 holds the cpu pointer. */ - bl cpu_main + bl cpu_entry /* Run the vcpu returned by cpu_main. */ bl vcpu_restore_all_and_run From 7de19be706ed2bc1978086db7ce36e00e03e157f Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 14:19:09 +0900 Subject: [PATCH 38/45] Remove singleton.rs and gather all singletons into `Hypervisor`. --- hfo2/src/api.rs | 30 ++++++++++++------------ hfo2/src/cpu.rs | 11 +++++---- hfo2/src/init.rs | 44 +++++++++++++++++++++++++---------- hfo2/src/mm.rs | 21 ++++++++++++----- hfo2/src/singleton.rs | 54 ------------------------------------------- hfo2/src/vm.rs | 8 +++---- 6 files changed, 72 insertions(+), 96 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 1c57aa3bb..925303287 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -24,10 +24,10 @@ use crate::abi::*; use crate::addr::*; use crate::arch::*; use crate::cpu::*; +use crate::init::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; -use crate::singleton::*; use crate::spci::*; use crate::spci_architected_message::*; use crate::spinlock::*; @@ -71,7 +71,7 @@ unsafe fn switch_to_primary( mut primary_ret: HfVCpuRunReturn, secondary_state: VCpuStatus, ) -> *mut VCpu { - let primary = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); + let primary = hafnium().vm_manager.get(HF_PRIMARY_VM_ID).unwrap(); let next = primary .vcpus .get(cpu_index(&*current.get_inner().cpu)) @@ -230,7 +230,7 @@ pub unsafe extern "C" fn api_vcpu_get_count( return 0; } - let vm = unwrap_or!(vm_manager().get(vm_id), return 0); + let vm = unwrap_or!(hafnium().vm_manager.get(vm_id), return 0); vm.vcpus.len() as spci_vcpu_count_t } @@ -386,7 +386,7 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested VM must exist. - let vm = ok_or_return!(vm_manager().get(vm_id).ok_or(()), ret.into_raw()); + let vm = ok_or_return!(hafnium().vm_manager.get(vm_id).ok_or(()), ret.into_raw()); // The requested vcpu must exist. let vcpu = ok_or_return!(vm.vcpus.get(vcpu_idx as usize).ok_or(()), ret.into_raw()); @@ -478,10 +478,7 @@ pub unsafe extern "C" fn api_vm_configure( // TODO: the scope of the can be reduced but will require restructing // to keep a single unlock point. let mut vm_inner = (*vm).inner.lock(); - if vm_inner - .configure(send, recv, &api_manager().ppool) - .is_err() - { + if vm_inner.configure(send, recv, &hafnium().mpool).is_err() { return -1; } @@ -536,7 +533,10 @@ pub unsafe extern "C" fn api_spci_msg_send( // Ensure the target VM exists. let to = ok_or_return!( - vm_manager().get(from_msg_replica.target_vm_id).ok_or(()), + hafnium() + .vm_manager + .get(from_msg_replica.target_vm_id) + .ok_or(()), SpciReturn::InvalidParameters ); @@ -715,7 +715,7 @@ pub unsafe extern "C" fn api_mailbox_waiter_get(vm_id: spci_vm_id_t, current: *c return -1; } - let vm = ok_or_return!(vm_manager().get(vm_id).ok_or(()), -1); + let vm = ok_or_return!(hafnium().vm_manager.get(vm_id).ok_or(()), -1); // Check if there are outstanding notifications from given vm. let entry = (*vm).inner.lock().fetch_waiter(); @@ -815,7 +815,7 @@ pub unsafe extern "C" fn api_interrupt_inject( next: *mut *mut VCpu, ) -> i64 { let mut current = ManuallyDrop::new(VCpuExecutionLocked::from_raw(current)); - let target_vm = ok_or_return!(vm_manager().get(target_vm_id).ok_or(()), -1); + let target_vm = ok_or_return!(hafnium().vm_manager.get(target_vm_id).ok_or(()), -1); if intid >= HF_NUM_INTIDS { return -1; @@ -841,7 +841,7 @@ pub unsafe extern "C" fn api_interrupt_inject( /// Clears a region of physical memory by overwriting it with zeros. The data is /// flushed from the cache so the memory has been cleared across the system. fn clear_memory(begin: paddr_t, end: paddr_t, ppool: &MPool) -> Result<(), ()> { - let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); + let mut hypervisor_ptable = hafnium().memory_manager.hypervisor_ptable.lock(); let size = pa_difference(begin, end); let region = pa_addr(begin); @@ -898,7 +898,7 @@ pub fn spci_share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool: MPool = MPool::new_with_fallback(&api_manager().ppool); + let local_page_pool: MPool = MPool::new_with_fallback(&hafnium().mpool); // Obtain the single contiguous set of pages from the memory_region. // TODO: Add support for multiple constituent regions. @@ -979,7 +979,7 @@ fn share_memory( } // Ensure the target VM exists. - let to = vm_manager().get(vm_id).ok_or(())?; + let to = hafnium().vm_manager.get(vm_id).ok_or(())?; let begin = addr; let end = ipa_add(addr, size); @@ -1004,7 +1004,7 @@ fn share_memory( // Create a local pool so any freed memory can't be used by another thread. // This is to ensure the original mapping can be restored if any stage of // the process fails. - let local_page_pool = MPool::new_with_fallback(&api_manager().ppool); + let local_page_pool = MPool::new_with_fallback(&hafnium().mpool); let (mut from_inner, mut to_inner) = SpinLock::lock_both(&(*from).inner, &(*to).inner); diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index e306f38dd..a75b43ab6 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -20,9 +20,9 @@ use core::ptr; use crate::addr::*; use crate::arch::*; +use crate::init::*; use crate::mm::*; use crate::page::*; -use crate::singleton::*; use crate::spinlock::*; use crate::std::*; use crate::types::*; @@ -429,7 +429,7 @@ impl CpuManager { *is_on = true; if !prev { - let vm = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); + let vm = hafnium().vm_manager.get(HF_PRIMARY_VM_ID).unwrap(); let vcpu = vm.vcpus.get(self.index_of(c)).unwrap(); vcpu.inner.lock().on(entry, arg); @@ -456,13 +456,13 @@ impl CpuManager { #[no_mangle] pub unsafe extern "C" fn cpu_index(c: *const Cpu) -> usize { - cpu_manager().index_of(&*c) + hafnium().cpu_manager.index_of(&*c) } /// Turns CPU on and returns the previous state. #[no_mangle] pub unsafe extern "C" fn cpu_on(c: *mut Cpu, entry: ipaddr_t, arg: uintreg_t) -> bool { - cpu_manager().cpu_on(&*c, entry, arg) + hafnium().cpu_manager.cpu_on(&*c, entry, arg) } /// Prepares the CPU for turning itself off. @@ -474,7 +474,8 @@ pub unsafe extern "C" fn cpu_off(c: *mut Cpu) { /// Searches for a CPU based on its id. #[no_mangle] pub unsafe extern "C" fn cpu_find(id: cpu_id_t) -> *mut Cpu { - cpu_manager() + hafnium() + .cpu_manager .lookup(id) .map(|cpu| cpu as *const _ as usize as *mut _) .unwrap_or(ptr::null_mut()) diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index 391373cbf..aaa848ff9 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -48,12 +48,23 @@ extern "C" { static boot_cpu: Cpu; } +/// TODO(HfO2): Refactor api.rs and remove `pub` (#44.) +pub struct Hypervisor { + pub mpool: MPool, + pub memory_manager: MemoryManager, + pub cpu_manager: CpuManager, + pub vm_manager: VmManager, +} + /// Note(HfO2): this variable was originally of type /// MaybeUninit<[u8; mem::size_of::() * HEAP_PAGES]>, /// but it was not aligned to PAGE_SIZE. static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); static mut INITED: bool = false; +/// TODO(HfO2): This `pub` is required by mm_init, which is only used by unit +/// tests. Resolving #46 may help. +pub static mut HAFNIUM: MaybeUninit = MaybeUninit::uninit(); /// Performs one-time initialisation of the hypervisor. #[no_mangle] @@ -68,6 +79,7 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { dlog!("Initialising hafnium\n"); arch_one_time_init(); + arch_cpu_module_init(); let mut ppool = MPool::new(); ppool.free_pages(Pages::from_raw( @@ -76,25 +88,28 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { )); let mm = MemoryManager::new(&ppool).expect("mm_init failed"); - memory_manager_init(mm); + + // We have to initialize memory manager early, because + // Stage2::invalidate_tlb refers global state. TODO(HfO2): Refactor the + // function to recieve the info from caller. Maybe MemoryManager should + // have a wrapper function for page table. + HAFNIUM.get_mut().memory_manager = mm; + let mm = &HAFNIUM.get_ref().memory_manager; // Enable locks now that mm is initialised. dlog_enable_lock(); mpool_enable_locks(); - let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); + let mut hypervisor_ptable = mm.hypervisor_ptable.lock(); let params = boot_params_get(&mut hypervisor_ptable, &mut ppool) .expect("unable to retrieve boot params"); - arch_cpu_module_init(); - let cpum = CpuManager::new( ¶ms.cpu_ids[..params.cpu_count], boot_cpu.id, &callstacks, ); - cpu_manager_init(cpum); for i in 0..params.mem_ranges_count { dlog!( @@ -121,11 +136,11 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { pa_difference(params.initrd_begin, params.initrd_end), ); - vm_manager_init(VmManager::new()); + HAFNIUM.get_mut().vm_manager = VmManager::new(); // Load all VMs. let primary_initrd = load_primary( - vm_manager_mut(), + &mut HAFNIUM.get_mut().vm_manager, &mut hypervisor_ptable, &cpio, params.kernel_arg, @@ -141,7 +156,7 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { ); load_secondary( - vm_manager_mut(), + &mut HAFNIUM.get_mut().vm_manager, &mut hypervisor_ptable, &cpio, ¶ms, @@ -156,8 +171,9 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { hypervisor_ptable.defrag(&ppool); - // Initialise the API page pool. ppool will be empty from now on. - api_manager_init(ApiManager::new(ppool)); + // Initialise HAFNIUM. + HAFNIUM.get_mut().mpool = ppool; + HAFNIUM.get_mut().cpu_manager = cpum; // Enable TLB invalidation for VM page table updates. mm_vm_enable_invalidation(); @@ -165,13 +181,17 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { dlog!("Hafnium initialisation completed\n"); INITED = true; - cpu_manager().boot_cpu() + hafnium().cpu_manager.boot_cpu() // From now on, other pCPUs are on in order to run multiple vCPUs. Thus // you may safely make readonly references to the Hafnium singleton, but // may not modify the singleton without proper locking. } +pub fn hafnium() -> &'static Hypervisor { + unsafe { HAFNIUM.get_ref() } +} + // The entry point of CPUs when they are turned on. It is supposed to initialise // all state and return the first vCPU to run. #[no_mangle] @@ -180,7 +200,7 @@ pub unsafe extern "C" fn cpu_main(c: *const Cpu) -> *mut VCpu { panic!("mm_cpu_init failed"); } - let primary = vm_manager().get(HF_PRIMARY_VM_ID).unwrap(); + let primary = hafnium().vm_manager.get(HF_PRIMARY_VM_ID).unwrap(); let vcpu = primary.vcpus.get(cpu_index(&*c)).unwrap(); let vm = vcpu.vm; diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index 9458e6a59..06a971f25 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -36,10 +36,10 @@ use reduce::Reduce; use crate::addr::*; use crate::arch::*; +use crate::init::*; use crate::layout::*; use crate::mpool::MPool; use crate::page::*; -use crate::singleton::*; use crate::spinlock::{SpinLock, SpinLockGuard}; use crate::types::*; use crate::utils::*; @@ -264,7 +264,11 @@ impl Stage for Stage2 { } fn invalidate_tlb(begin: usize, end: usize) { - if memory_manager().stage2_invalidate.load(Ordering::Relaxed) { + if hafnium() + .memory_manager + .stage2_invalidate + .load(Ordering::Relaxed) + { unsafe { arch_mm_invalidate_stage2_range(ipa_init(begin), ipa_init(end)); } @@ -1101,7 +1105,8 @@ impl MemoryManager { /// This function should not be invoked concurrently with other memory management functions. #[no_mangle] pub unsafe extern "C" fn mm_vm_enable_invalidation() { - memory_manager() + hafnium() + .memory_manager .stage2_invalidate .store(true, Ordering::Relaxed); } @@ -1238,14 +1243,18 @@ pub unsafe extern "C" fn mm_unmap( #[no_mangle] pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { let mm = ok_or_return!(MemoryManager::new(&*mpool).ok_or(()), false); - memory_manager_init(mm); + HAFNIUM.get_mut().memory_manager = mm; true } #[no_mangle] pub unsafe extern "C" fn mm_cpu_init() -> bool { - let raw_ptable = memory_manager().hypervisor_ptable.get_mut_unchecked().root; + let raw_ptable = hafnium() + .memory_manager + .hypervisor_ptable + .get_mut_unchecked() + .root; arch_mm_init(raw_ptable, false) } @@ -1257,7 +1266,7 @@ pub unsafe extern "C" fn mm_defrag(mut stage1_locked: mm_stage1_locked, mpool: * #[no_mangle] pub unsafe extern "C" fn mm_lock_stage1() -> mm_stage1_locked { - let ptable = &memory_manager().hypervisor_ptable; + let ptable = &hafnium().memory_manager.hypervisor_ptable; ptable.lock().into() } diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs index d9982dde0..8b1b37063 100644 --- a/hfo2/src/singleton.rs +++ b/hfo2/src/singleton.rs @@ -45,57 +45,3 @@ //! Setting singleton getters as safe causes one problem, that calling one of //! those functions in initalizing sequence is dangerous if the singleton were //! not yet initialized. Watch your code! - -use core::mem::MaybeUninit; - -use crate::api::ApiManager; -use crate::cpu::CpuManager; -use crate::mm::MemoryManager; -use crate::vm::VmManager; - -// Singletons. -static mut MEMORY_MANAGER: MaybeUninit = MaybeUninit::uninit(); -static mut VM_MANAGER: MaybeUninit = MaybeUninit::uninit(); -static mut API_MANAGER: MaybeUninit = MaybeUninit::uninit(); -static mut CPU_MANAGER: MaybeUninit = MaybeUninit::uninit(); - -// Singleton initializers. -pub unsafe fn memory_manager_init(mm: MemoryManager) { - MEMORY_MANAGER = MaybeUninit::new(mm); -} - -pub unsafe fn vm_manager_init(vmm: VmManager) { - VM_MANAGER = MaybeUninit::new(vmm); -} - -pub unsafe fn api_manager_init(apim: ApiManager) { - API_MANAGER = MaybeUninit::new(apim); -} - -pub unsafe fn cpu_manager_init(cpum: CpuManager) { - CPU_MANAGER = MaybeUninit::new(cpum); -} - -// Global functions to acquire read-only references to singletons. -pub fn memory_manager() -> &'static MemoryManager { - unsafe { MEMORY_MANAGER.get_ref() } -} - -pub fn vm_manager() -> &'static VmManager { - unsafe { VM_MANAGER.get_ref() } -} - -pub fn api_manager() -> &'static ApiManager { - unsafe { API_MANAGER.get_ref() } -} - -pub fn cpu_manager() -> &'static CpuManager { - unsafe { CPU_MANAGER.get_ref() } -} - -/// A temporal function to load VM correctly. Today VM need to know its address -/// to initialize itself, thus we cannot simply 'move' the initialized VmManager -/// into VM_MANAGER. TODO(HfO2): Fix it, and remove this function. -pub unsafe fn vm_manager_mut() -> &'static mut VmManager { - VM_MANAGER.get_mut() -} diff --git a/hfo2/src/vm.rs b/hfo2/src/vm.rs index 2b4432687..94d32311f 100644 --- a/hfo2/src/vm.rs +++ b/hfo2/src/vm.rs @@ -26,11 +26,11 @@ use scopeguard::guard; use crate::addr::*; use crate::arch::*; use crate::cpu::*; +use crate::init::*; use crate::list::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; -use crate::singleton::*; use crate::spci::*; use crate::spinlock::*; use crate::std::*; @@ -156,7 +156,7 @@ impl Mailbox { ) -> Result<(), ()> { // TODO(HfO2): Acquring the singleton here is not recommended. Get the // hypervisor ptable from callee (API module.) - let mut hypervisor_ptable = memory_manager().hypervisor_ptable.lock(); + let mut hypervisor_ptable = hafnium().memory_manager.hypervisor_ptable.lock(); let mut ptable = guard(hypervisor_ptable.deref_mut(), |_| ()); // Map the send page as read-only in the hypervisor address space. @@ -616,7 +616,7 @@ pub unsafe extern "C" fn vm_init( ppool: *mut MPool, new_vm: *mut *mut Vm, ) -> bool { - let vmm = vm_manager() as *const _ as usize as *mut VmManager; + let vmm = &hafnium().vm_manager as *const _ as usize as *mut VmManager; match (*vmm).new_vm(vcpu_count, &*ppool) { Some(vm) => { *new_vm = vm as *mut _; @@ -628,7 +628,7 @@ pub unsafe extern "C" fn vm_init( #[no_mangle] pub unsafe extern "C" fn vm_get_count() -> spci_vm_count_t { - vm_manager().vms.len() as _ + hafnium().vm_manager.vms.len() as _ } /// Locks the given VM and updates `locked` to hold the newly locked vm. From 04dbdf88adc6b780e9fa8b322d1ef41be40fd74c Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 16:26:35 +0900 Subject: [PATCH 39/45] Use ptr::write not to drop PageTable, and remove unnecessary stuffs. --- hfo2/src/api.rs | 13 ------------ hfo2/src/init.rs | 20 +++++++++++++++--- hfo2/src/lib.rs | 1 - hfo2/src/mm.rs | 2 +- hfo2/src/singleton.rs | 47 ------------------------------------------- 5 files changed, 18 insertions(+), 65 deletions(-) delete mode 100644 hfo2/src/singleton.rs diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 925303287..efdfcfbac 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -48,19 +48,6 @@ use crate::vm::*; // of a page. const_assert_eq!(hf_mailbox_size; HF_MAILBOX_SIZE, PAGE_SIZE); -pub struct ApiManager { - /// A global page pool for sharing memories. - ppool: MPool, -} - -impl ApiManager { - /// Initialises the API page pool by taking ownership of the contents of the - /// given page pool. - pub fn new(ppool: MPool) -> Self { - Self { ppool } - } -} - /// Switches the physical CPU back to the corresponding vcpu of the primary VM. /// /// This triggers the scheduling logic to run. Run in the context of secondary diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index aaa848ff9..a9fe63566 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -17,7 +17,6 @@ use core::mem::MaybeUninit; use crate::addr::*; -use crate::api::*; use crate::arch::*; use crate::boot_params::*; use crate::cpu::*; @@ -26,7 +25,6 @@ use crate::memiter::*; use crate::mm::*; use crate::mpool::*; use crate::page::*; -use crate::singleton::*; use crate::types::*; use crate::vm::*; @@ -62,8 +60,24 @@ pub struct Hypervisor { static mut PTABLE_BUF: MaybeUninit<[RawPage; HEAP_PAGES]> = MaybeUninit::uninit(); static mut INITED: bool = false; + +/// A singleton collecting all managers in Hafnium. +/// +/// This is dependency-free; Typical solutions of mutable and shared static +/// objects delay their initialization. Considering concurrency, they often use +/// std::sync features to prevent racy initialization. But Hafnium is different. +/// +/// - The initialization is _always_ happened once in the specific time. +/// - During the time, no other thread is running; Hafnium runs as if it were +/// a single-thread program. +/// - After the initialization, Hafnium may make a non-exclusive reference of +/// singletons, but they have their own way for Hafnium to safely write them. +/// +/// Therefore, I do not use a safe wrapper for initialization such as +/// `std::sync::Once` and `lazy_static`. +/// /// TODO(HfO2): This `pub` is required by mm_init, which is only used by unit -/// tests. Resolving #46 may help. +/// tests. Resolving #46 may help to remove the `pub`. pub static mut HAFNIUM: MaybeUninit = MaybeUninit::uninit(); /// Performs one-time initialisation of the hypervisor. diff --git a/hfo2/src/lib.rs b/hfo2/src/lib.rs index 61b1e8dcd..cd8369dfe 100644 --- a/hfo2/src/lib.rs +++ b/hfo2/src/lib.rs @@ -56,7 +56,6 @@ mod mm; mod mpool; mod page; mod panic; -mod singleton; mod slist; mod spci; mod spci_architected_message; diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index 06a971f25..fb012c305 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -1243,7 +1243,7 @@ pub unsafe extern "C" fn mm_unmap( #[no_mangle] pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { let mm = ok_or_return!(MemoryManager::new(&*mpool).ok_or(()), false); - HAFNIUM.get_mut().memory_manager = mm; + ptr::write(&mut HAFNIUM.get_mut().memory_manager, mm); true } diff --git a/hfo2/src/singleton.rs b/hfo2/src/singleton.rs deleted file mode 100644 index 8b1b37063..000000000 --- a/hfo2/src/singleton.rs +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2019 Sanguk Park - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! A module collecting all the singleton objects in Hafnium. -//! -//! This module is dependency-free; Typical solutions of mutable and shared -//! static objects delay their initialization. Considering concurrency, they -//! often use std::sync features to prevent racy initialization. But Hafnium is -//! different. -//! -//! - The initialization is _always_ happened once in the specific time. -//! - During the time, no other thread is running; Hafnium runs as if it were -//! a single-thread program. -//! - After the initialization, Hafnium may make a non-exclusive reference of -//! singletons, but they have their own way for Hafnium to safely write them. -//! -//! Therefore, I do not use a safe wrapper for initialization such as -//! `std::sync::Once` and `lazy_static`. -//! -//! # Safety -//! -//! Hafnium has two states representing its overall life. -//! - Initializing. -//! - Exception handling. -//! -//! Hafnium starts from initializing but eventually moves into exception -//! handling state. Thus I treat exception handling state as if it were -//! Hafnium's _usual_ mode; that's why `{memory, vm, api, cpu}_manager` -//! functions are not denoted by `unsafe`. On the other hand, initializers must -//! not be called in the handling state, so that they are `unsafe`. -//! -//! Setting singleton getters as safe causes one problem, that calling one of -//! those functions in initalizing sequence is dangerous if the singleton were -//! not yet initialized. Watch your code! From 2c701098e232b17c0c511c7e78163e23be669f83 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 16:30:44 +0900 Subject: [PATCH 40/45] Use ptr::write more. --- hfo2/src/init.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index a9fe63566..d67716def 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -15,6 +15,7 @@ */ use core::mem::MaybeUninit; +use core::ptr; use crate::addr::*; use crate::arch::*; @@ -107,7 +108,7 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { // Stage2::invalidate_tlb refers global state. TODO(HfO2): Refactor the // function to recieve the info from caller. Maybe MemoryManager should // have a wrapper function for page table. - HAFNIUM.get_mut().memory_manager = mm; + ptr::write(&mut HAFNIUM.get_mut().memory_manager, mm); let mm = &HAFNIUM.get_ref().memory_manager; // Enable locks now that mm is initialised. @@ -150,7 +151,7 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { pa_difference(params.initrd_begin, params.initrd_end), ); - HAFNIUM.get_mut().vm_manager = VmManager::new(); + ptr::write(&mut HAFNIUM.get_mut().vm_manager, VmManager::new()); // Load all VMs. let primary_initrd = load_primary( @@ -186,8 +187,8 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { hypervisor_ptable.defrag(&ppool); // Initialise HAFNIUM. - HAFNIUM.get_mut().mpool = ppool; - HAFNIUM.get_mut().cpu_manager = cpum; + ptr::write(&mut HAFNIUM.get_mut().mpool, ppool); + ptr::write(&mut HAFNIUM.get_mut().cpu_manager, cpum); // Enable TLB invalidation for VM page table updates. mm_vm_enable_invalidation(); From 0180a56298771467112335fca6b0f3fd7c97254c Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 17:32:04 +0900 Subject: [PATCH 41/45] Do not initialize ArchRegs when created, to prevent stack overflow. --- hfo2/src/cpu.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index a75b43ab6..1d722c77c 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -203,17 +203,22 @@ impl Interrupts { impl ArchRegs { pub fn new() -> Self { - let mut ret; - unsafe { - ret = MaybeUninit::uninit().assume_init(); - memset_s( - &mut ret as *mut _ as usize as *mut _, - mem::size_of_val(&ret), - 0, - mem::size_of_val(&ret), - ); - } - ret + // TODO(HfO2): Originally, ArchRegs are filled by 0 when they're crated. + // However, doing like + // ``` + // let mut ret = MaybeUninit::uninit().assume_init(); + // memset(&mut ret, 0, mem::size_of_val(&ret)); + // ``` + // allocates large memory in stack to hold the value of ArchRegs, and + // introduce unnecessary memcpy. + // Note that MaybeUninit::zeroed() makes same assembly. I guess Rust + // native MaybeUninit can be improved to generate higher quality code. + // + // Anyway, none of them works, thus we delay its initialization to the + // time when `reset` is called. Today Hafnium's implementation always + // call `reset` before using `ArchRegs`. But, the future is in shadow, + // we'd better refactor this, by making arch-dependent `new` methods. + unsafe { MaybeUninit::uninit().assume_init() } } /// Reset the register values other than the PC and argument which are set From 8de583df74cb1161fdaec7196e83070ea895bf51 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 17:54:14 +0900 Subject: [PATCH 42/45] Checks cpu_ids contains zero or multiple boot CPU IDs, and make a TODO. --- hfo2/src/cpu.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hfo2/src/cpu.rs b/hfo2/src/cpu.rs index 1d722c77c..954c35a27 100644 --- a/hfo2/src/cpu.rs +++ b/hfo2/src/cpu.rs @@ -24,7 +24,6 @@ use crate::init::*; use crate::mm::*; use crate::page::*; use crate::spinlock::*; -use crate::std::*; use crate::types::*; use crate::vm::*; @@ -410,6 +409,13 @@ impl CpuManager { let boot_stack = stacks[0].as_ptr() as usize; cpus.push(Cpu::new(boot_cpu_id, boot_stack + STACK_SIZE, true)); + // TODO(HfO2): Ask hafnium-discuss about zero or multiple boot CPU IDs + // and the reason why Hafnium initializes pCPUs in reverse order. If it + // has a special reason, fix this (#51.) + if cpu_ids.iter().filter(|id| boot_cpu_id == **id).count() != 1 { + panic!("`cpu_ids` contains zero or multiple boot CPU IDs.\n"); + } + let cpu_ids = cpu_ids.iter().filter(|id| boot_cpu_id != **id); let stacks = stacks.iter().skip(1); From 31b1169d39e0a73391ff52076e3d777032492241 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 18:11:07 +0900 Subject: [PATCH 43/45] Use ok_or! / unwrap_or! --- hfo2/src/api.rs | 39 +++++++++++++++++++++------------------ hfo2/src/fdt.rs | 19 +++++++++---------- hfo2/src/fdt_handler.rs | 8 ++++---- hfo2/src/load.rs | 22 ++++++++-------------- hfo2/src/mm.rs | 14 +++++++------- hfo2/src/utils.rs | 4 ++-- 6 files changed, 51 insertions(+), 55 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index efdfcfbac..4e8514464 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -373,16 +373,19 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested VM must exist. - let vm = ok_or_return!(hafnium().vm_manager.get(vm_id).ok_or(()), ret.into_raw()); + let vm = unwrap_or!( + hafnium().vm_manager.get(vm_id), + return ret.into_raw() + ); // The requested vcpu must exist. - let vcpu = ok_or_return!(vm.vcpus.get(vcpu_idx as usize).ok_or(()), ret.into_raw()); + let vcpu = unwrap_or!( + vm.vcpus.get(vcpu_idx as usize), + return ret.into_raw() + ); // Update state if allowed. - let mut vcpu_locked = match vcpu_prepare_run(¤t, vcpu, ret) { - Ok(locked) => locked, - Err(ret) => return ret.into_raw(), - }; + let mut vcpu_locked = ok_or!(vcpu_prepare_run(¤t, vcpu, ret), return ret.into_raw()); // Inject timer interrupt if timer has expired. It's safe to access // vcpu->regs here because vcpu_prepare_run already made sure that @@ -519,12 +522,9 @@ pub unsafe extern "C" fn api_spci_msg_send( } // Ensure the target VM exists. - let to = ok_or_return!( - hafnium() - .vm_manager - .get(from_msg_replica.target_vm_id) - .ok_or(()), - SpciReturn::InvalidParameters + let to = unwrap_or!( + hafnium().vm_manager.get(from_msg_replica.target_vm_id), + return SpciReturn::InvalidParameters ); // Hf needs to hold the lock on `to` before the mailbox state is checked. @@ -702,7 +702,7 @@ pub unsafe extern "C" fn api_mailbox_waiter_get(vm_id: spci_vm_id_t, current: *c return -1; } - let vm = ok_or_return!(hafnium().vm_manager.get(vm_id).ok_or(()), -1); + let vm = unwrap_or!(hafnium().vm_manager.get(vm_id), return -1); // Check if there are outstanding notifications from given vm. let entry = (*vm).inner.lock().fetch_waiter(); @@ -802,7 +802,7 @@ pub unsafe extern "C" fn api_interrupt_inject( next: *mut *mut VCpu, ) -> i64 { let mut current = ManuallyDrop::new(VCpuExecutionLocked::from_raw(current)); - let target_vm = ok_or_return!(hafnium().vm_manager.get(target_vm_id).ok_or(()), -1); + let target_vm = unwrap_or!(hafnium().vm_manager.get(target_vm_id), return -1); if intid >= HF_NUM_INTIDS { return -1; @@ -812,7 +812,10 @@ pub unsafe extern "C" fn api_interrupt_inject( return -1; } - let target_vcpu = ok_or_return!(target_vm.vcpus.get(target_vcpu_idx as usize).ok_or(()), -1); + let target_vcpu = unwrap_or!( + target_vm.vcpus.get(target_vcpu_idx as usize), + return -1 + ); dlog!( "Injecting IRQ {} for VM {} VCPU {} from VM {} VCPU {}\n", @@ -897,7 +900,7 @@ pub fn spci_share_memory( // Check if the state transition is lawful for both VMs involved in the // memory exchange, ensure that all constituents of a memory region being // shared are at the same state. - let (orig_from_mode, from_mode, to_mode) = ok_or_return!( + let (orig_from_mode, from_mode, to_mode) = ok_or!( spci_msg_check_transition( &to_locked, &from_locked, @@ -906,7 +909,7 @@ pub fn spci_share_memory( end, memory_to_attributes, ), - SpciReturn::InvalidParameters + return SpciReturn::InvalidParameters ); let pa_begin = pa_from_ipa(begin); @@ -1070,7 +1073,7 @@ pub unsafe extern "C" fn api_share_memory( ) -> i64 { // Convert the sharing request to memory management modes. // The input is untrusted so might not be a valid value. - let share = ok_or_return!(HfShare::try_from(share), -1); + let share = ok_or!(HfShare::try_from(share), return -1); match share_memory(vm_id, addr, size, share, &*current) { Ok(_) => 0, diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 45cfbe9db..14dd3728d 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -193,15 +193,14 @@ impl FdtTokenizer { return None; } - match self.str() { - Some(name) => Some(name), - None => { - // Move cursor to the end so that caller won't get any new - // tokens. - self.cur = self.end; - None - } - } + let name = unwrap_or!(self.str(), { + // Move cursor to the end so that caller won't get any new + // tokens. + self.cur = self.end; + return None + }); + + Some(name) } unsafe fn skip_properties(&mut self) { @@ -458,7 +457,7 @@ pub unsafe extern "C" fn fdt_read_property( buf: *mut *const u8, size: *mut u32, ) -> bool { - let (prop_buf, prop_size) = ok_or_return!((*node).read_property(name), false); + let (prop_buf, prop_size) = ok_or!((*node).read_property(name), return false); *buf = prop_buf; *size = prop_size; true diff --git a/hfo2/src/fdt_handler.rs b/hfo2/src/fdt_handler.rs index a72b4ee9a..3e6cc3c5c 100644 --- a/hfo2/src/fdt_handler.rs +++ b/hfo2/src/fdt_handler.rs @@ -72,14 +72,14 @@ impl FdtNode { return false; } - let initrd_begin = ok_or_return!(self.read_number("linux,initrd-start\0".as_ptr()), { + let initrd_begin = ok_or!(self.read_number("linux,initrd-start\0".as_ptr()), { dlog!("Unable to read linux,initrd-start\n"); - false + return false; }); - let initrd_end = ok_or_return!(self.read_number("linux,initrd-end\0".as_ptr()), { + let initrd_end = ok_or!(self.read_number("linux,initrd-end\0".as_ptr()), { dlog!("Unable to read linux,initrd-end\n"); - false + return false; }); *begin = pa_init(initrd_begin as usize); diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 733fc5e3a..9ffd6aad7 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -252,13 +252,10 @@ pub unsafe fn load_secondary( } let (secondary_mem_begin, secondary_mem_end) = - match carve_out_mem_range(&mut mem_ranges_available, mem as u64) { - Ok(range) => range, - Err(_) => { - dlog!("Not enough memory ({} bytes)\n", mem); - continue; - } - }; + ok_or!(carve_out_mem_range(&mut mem_ranges_available, mem as u64), { + dlog!("Not enough memory ({} bytes)\n", mem); + continue; + }); if !copy_to_unmapped( hypervisor_ptable, @@ -285,13 +282,10 @@ pub unsafe fn load_secondary( return Err(()); } - let vm = match vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool) { - Some(vm) => vm, - None => { - dlog!("Unable to initialise VM\n"); - continue; - } - }; + let vm = unwrap_or!(vm_manager.new_vm(cpu as spci_vcpu_count_t, ppool), { + dlog!("Unable to initialise VM\n"); + continue; + }); // Grant the VM access to the memory. if vm diff --git a/hfo2/src/mm.rs b/hfo2/src/mm.rs index fb012c305..4c84899b0 100644 --- a/hfo2/src/mm.rs +++ b/hfo2/src/mm.rs @@ -1173,17 +1173,17 @@ pub unsafe extern "C" fn mm_vm_unmap_hypervisor( // TODO: If we add pages dynamically, they must be included here too. let t = &mut *t; let mpool = &*mpool; - ok_or_return!( + ok_or!( t.unmap(layout_text_begin(), layout_text_end(), mpool), - false + return false ); - ok_or_return!( + ok_or!( t.unmap(layout_rodata_begin(), layout_rodata_end(), mpool), - false + return false ); - ok_or_return!( + ok_or!( t.unmap(layout_data_begin(), layout_data_end(), mpool), - false + return false ); true } @@ -1242,7 +1242,7 @@ pub unsafe extern "C" fn mm_unmap( /// Unsafety doesn't really matter. #[no_mangle] pub unsafe extern "C" fn mm_init(mpool: *const MPool) -> bool { - let mm = ok_or_return!(MemoryManager::new(&*mpool).ok_or(()), false); + let mm = unwrap_or!(MemoryManager::new(&*mpool), return false); ptr::write(&mut HAFNIUM.get_mut().memory_manager, mm); true diff --git a/hfo2/src/utils.rs b/hfo2/src/utils.rs index 8114d5b70..5814e55b8 100644 --- a/hfo2/src/utils.rs +++ b/hfo2/src/utils.rs @@ -17,11 +17,11 @@ use core::sync::atomic::spin_loop_hint; #[macro_export] -macro_rules! ok_or_return { +macro_rules! ok_or { ($e:expr, $err:expr) => {{ match $e { Ok(r) => r, - Err(_) => return $err, + Err(_) => $err, } }}; } From 4c5395e7af5416e6bd7de6817a30cc5a9511ef88 Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 18:31:29 +0900 Subject: [PATCH 44/45] Revert a part of 31b1169, that was wrong. --- hfo2/src/api.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 4e8514464..64e0e9b4d 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -385,7 +385,10 @@ pub unsafe extern "C" fn api_vcpu_run( ); // Update state if allowed. - let mut vcpu_locked = ok_or!(vcpu_prepare_run(¤t, vcpu, ret), return ret.into_raw()); + let mut vcpu_locked = match vcpu_prepare_run(¤t, vcpu, ret) { + Ok(locked) => locked, + Err(ret) => return ret.into_raw(), + }; // Inject timer interrupt if timer has expired. It's safe to access // vcpu->regs here because vcpu_prepare_run already made sure that From 3d5c7c8fc6a1ce57583893b1f1e060eab127629b Mon Sep 17 00:00:00 2001 From: "Park, Sanguk" Date: Wed, 11 Sep 2019 18:49:31 +0900 Subject: [PATCH 45/45] Make basic getters and util functions of MemIter. Remove `pub`s of member fields on MemIter. --- hfo2/src/api.rs | 15 +++------------ hfo2/src/fdt.rs | 4 ++-- hfo2/src/init.rs | 4 ++-- hfo2/src/load.rs | 28 +++++++++++++--------------- hfo2/src/memiter.rs | 21 +++++++++++++++++++-- 5 files changed, 39 insertions(+), 33 deletions(-) diff --git a/hfo2/src/api.rs b/hfo2/src/api.rs index 64e0e9b4d..0dbdaf3aa 100644 --- a/hfo2/src/api.rs +++ b/hfo2/src/api.rs @@ -373,16 +373,10 @@ pub unsafe extern "C" fn api_vcpu_run( } // The requested VM must exist. - let vm = unwrap_or!( - hafnium().vm_manager.get(vm_id), - return ret.into_raw() - ); + let vm = unwrap_or!(hafnium().vm_manager.get(vm_id), return ret.into_raw()); // The requested vcpu must exist. - let vcpu = unwrap_or!( - vm.vcpus.get(vcpu_idx as usize), - return ret.into_raw() - ); + let vcpu = unwrap_or!(vm.vcpus.get(vcpu_idx as usize), return ret.into_raw()); // Update state if allowed. let mut vcpu_locked = match vcpu_prepare_run(¤t, vcpu, ret) { @@ -815,10 +809,7 @@ pub unsafe extern "C" fn api_interrupt_inject( return -1; } - let target_vcpu = unwrap_or!( - target_vm.vcpus.get(target_vcpu_idx as usize), - return -1 - ); + let target_vcpu = unwrap_or!(target_vm.vcpus.get(target_vcpu_idx as usize), return -1); dlog!( "Injecting IRQ {} for VM {} VCPU {} from VM {} VCPU {}\n", diff --git a/hfo2/src/fdt.rs b/hfo2/src/fdt.rs index 14dd3728d..5c0411c48 100644 --- a/hfo2/src/fdt.rs +++ b/hfo2/src/fdt.rs @@ -197,9 +197,9 @@ impl FdtTokenizer { // Move cursor to the end so that caller won't get any new // tokens. self.cur = self.end; - return None + return None; }); - + Some(name) } diff --git a/hfo2/src/init.rs b/hfo2/src/init.rs index d67716def..a346b14c6 100644 --- a/hfo2/src/init.rs +++ b/hfo2/src/init.rs @@ -166,8 +166,8 @@ unsafe extern "C" fn one_time_init(c: *mut Cpu) -> *mut Cpu { // load_secondary will add regions assigned to the secondary VMs from // mem_ranges to reserved_ranges. let mut update: BootParamsUpdate = BootParamsUpdate::new( - pa_from_va(va_from_ptr(primary_initrd.next as usize as *const _)), - pa_from_va(va_from_ptr(primary_initrd.limit as usize as *const _)), + pa_from_va(va_from_ptr(primary_initrd.get_next() as usize as *const _)), + pa_from_va(va_from_ptr(primary_initrd.get_limit() as usize as *const _)), ); load_secondary( diff --git a/hfo2/src/load.rs b/hfo2/src/load.rs index 9ffd6aad7..b3d09e69f 100644 --- a/hfo2/src/load.rs +++ b/hfo2/src/load.rs @@ -15,6 +15,7 @@ */ use core::mem; +use core::str; use crate::addr::*; use crate::arch::*; @@ -86,8 +87,8 @@ pub unsafe fn load_primary( if !copy_to_unmapped( hypervisor_ptable, primary_begin, - it.next as usize as *mut _, - it.limit.offset_from(it.next) as usize, + it.get_next() as usize as *mut _, + it.len(), ppool, ) { dlog!("Unable to relocate kernel for primary vm.\n"); @@ -229,14 +230,9 @@ pub unsafe fn load_secondary( let mut mem = unwrap_or!(it.parse_uint(), break); let cpu = unwrap_or!(it.parse_uint(), break); let name = unwrap_or!(it.parse_str(), break); + let name_str = str::from_utf8_unchecked(name.as_slice()); - dlog!("Loading "); - let mut p = name.next; - while p != name.limit { - dlog!("{}", *p as char); - p = p.add(1); - } - dlog!("\n"); + dlog!("Loading {}\n", name_str); let kernel = unwrap_or!(find_file_memiter(&mut cpio.clone(), &name), { dlog!("Unable to load kernel\n"); @@ -246,22 +242,24 @@ pub unsafe fn load_secondary( // Round up to page size. mem = (mem + PAGE_SIZE as u64 - 1) & !(PAGE_SIZE as u64 - 1); - if mem < kernel.limit.offset_from(kernel.next) as u64 { + if mem < kernel.len() as u64 { dlog!("Kernel is larger than available memory\n"); continue; } - let (secondary_mem_begin, secondary_mem_end) = - ok_or!(carve_out_mem_range(&mut mem_ranges_available, mem as u64), { + let (secondary_mem_begin, secondary_mem_end) = ok_or!( + carve_out_mem_range(&mut mem_ranges_available, mem as u64), + { dlog!("Not enough memory ({} bytes)\n", mem); continue; - }); + } + ); if !copy_to_unmapped( hypervisor_ptable, secondary_mem_begin, - kernel.next as usize as *const _, - kernel.limit.offset_from(kernel.next) as usize, + kernel.get_next() as usize as *const _, + kernel.len(), ppool, ) { dlog!("Unable to copy kernel\n"); diff --git a/hfo2/src/memiter.rs b/hfo2/src/memiter.rs index 980a5de78..8af99bce7 100644 --- a/hfo2/src/memiter.rs +++ b/hfo2/src/memiter.rs @@ -15,6 +15,7 @@ */ use core::ptr; +use core::slice; use crate::std::*; use crate::types::*; @@ -22,8 +23,8 @@ use crate::types::*; #[repr(C)] #[derive(Clone)] pub struct MemIter { - pub next: *const u8, - pub limit: *const u8, + next: *const u8, + limit: *const u8, } /// Determines if a character is a whitespace. @@ -145,6 +146,22 @@ impl MemIter { None } } + + pub fn get_next(&self) -> *const u8 { + self.next + } + + pub fn get_limit(&self) -> *const u8 { + self.limit + } + + pub unsafe fn as_slice(&self) -> &[u8] { + slice::from_raw_parts(self.next, self.limit.offset_from(self.next) as usize) + } + + pub fn len(&self) -> usize { + unsafe { self.limit.offset_from(self.next) as usize } + } } #[no_mangle]