diff --git a/modules/axdma/src/dma.rs b/modules/axdma/src/dma.rs index 81ae42c97f..a1eae85f76 100644 --- a/modules/axdma/src/dma.rs +++ b/modules/axdma/src/dma.rs @@ -2,10 +2,7 @@ use core::{alloc::Layout, ptr::NonNull}; use allocator::{AllocError, AllocResult, BaseAllocator, ByteAllocator}; use axalloc::{DefaultByteAllocator, global_allocator}; -use axhal::{ - mem::virt_to_phys, - paging::{MappingFlags, PageSize}, -}; +use axhal::{mem::virt_to_phys, paging::MappingFlags}; use kspin::SpinNoIrq; use log::{debug, error}; use memory_addr::{PAGE_SIZE_4K, VirtAddr, va}; @@ -97,7 +94,7 @@ impl DmaAllocator { let expand_size = num_pages * PAGE_SIZE_4K; axmm::kernel_aspace() .lock() - .protect(vaddr, expand_size, flags, PageSize::Size4K) + .protect(vaddr, expand_size, flags) .map_err(|e| { error!("change table flag fail: {e:?}"); AllocError::NoMemory diff --git a/modules/axhal/src/arch/x86_64/trap.rs b/modules/axhal/src/arch/x86_64/trap.rs index 55c42e44b4..c777e90a35 100644 --- a/modules/axhal/src/arch/x86_64/trap.rs +++ b/modules/axhal/src/arch/x86_64/trap.rs @@ -78,6 +78,7 @@ fn vec_to_str(vec: u64) -> &'static str { fn err_code_to_flags(err_code: u64) -> Result { let code = PageFaultErrorCode::from_bits_truncate(err_code); let reserved_bits = (PageFaultErrorCode::CAUSED_BY_WRITE + | PageFaultErrorCode::PROTECTION_VIOLATION | PageFaultErrorCode::USER_MODE | PageFaultErrorCode::INSTRUCTION_FETCH) .complement(); diff --git a/modules/axmm/Cargo.toml b/modules/axmm/Cargo.toml index 46a7d5d828..4a84a0282e 100644 --- a/modules/axmm/Cargo.toml +++ b/modules/axmm/Cargo.toml @@ -9,11 +9,16 @@ homepage.workspace = true repository = "https://github.com/arceos-org/arceos/tree/main/modules/axmm" documentation = "https://arceos-org.github.io/arceos/axmm/index.html" +[features] +default = [] +cow = [] + [dependencies] axhal = { workspace = true, features = ["paging"] } axalloc = { workspace = true } axconfig = { workspace = true } +lazy_static = { version = "1.5", features = ["spin_no_std"] } log = "=0.4.21" axerrno = "0.1" lazyinit = "0.2" diff --git a/modules/axmm/src/aspace.rs b/modules/axmm/src/aspace.rs index f25c7d7b53..c839eab5b8 100644 --- a/modules/axmm/src/aspace.rs +++ b/modules/axmm/src/aspace.rs @@ -3,14 +3,18 @@ use core::fmt; use axerrno::{AxError, AxResult, ax_err}; use axhal::mem::phys_to_virt; use axhal::paging::{MappingFlags, PageTable, PagingError}; -use memory_addr::{ - MemoryAddr, PAGE_SIZE_4K, PageIter4K, PhysAddr, VirtAddr, VirtAddrRange, is_aligned, -}; +use memory_addr::{MemoryAddr, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange, is_aligned}; use memory_set::{MemoryArea, MemorySet}; use page_table_multiarch::PageSize; -use crate::backend::{Backend, PageIterWrapper}; +use crate::backend::Backend; use crate::mapping_err_to_ax_err; +use crate::page_iter_wrapper::PageIterWrapper; + +#[cfg(feature = "cow")] +use crate::backend::{alloc_frame, dealloc_frame}; +#[cfg(feature = "cow")] +use crate::frameinfo::frame_table; /// The virtual memory address space. pub struct AddrSpace { @@ -221,43 +225,96 @@ impl AddrSpace { Ok(()) } - /// Populates the area with physical frames, returning false if the area - /// contains unmapped area. - pub fn populate_area(&mut self, mut start: VirtAddr, size: usize, align: PageSize) -> AxResult { - self.validate_region(start, size, align)?; + /// Ensures that the specified virtual memory region is fully mapped. + /// + /// This function walks through the given virtual address range and attempts to ensure + /// that every page is mapped. If a page is not mapped and the corresponding area allows + /// on-demand population (`populate == false`), it will trigger a page fault to map it. + /// If `access_flags` contains `WRITE`, it will handle copy-on-write (COW) logic for already + /// mapped pages that may require COW due to write intentions. + /// + /// # Parameters + /// + /// - `start`: The starting virtual address of the region to map. + /// - `size`: The size (in bytes) of the region. + /// - `access_flags` indicates the access type + /// + /// # Returns + /// + /// Returns `Ok(())` if the entire region is successfully mapped, or an appropriate + /// `AxError` variant (`NoMemory`, `BadAddress`) on failure. + /// + /// # Errors + /// + /// - `AxError::NoMemory`: Failed to allocate. + /// - `AxError::BadAddress`: An invalid mapping state was detected. + pub fn populate_area( + &mut self, + mut start: VirtAddr, + size: usize, + _access_flags: MappingFlags, + ) -> AxResult { + self.validate_region(start, size, PageSize::Size4K)?; let end = start + size; - while let Some(area) = self.areas.find(start) { + for area in self.areas.iter() { + if start >= area.end() { + continue; + } + + if start < area.start() { + // If the area is not fully mapped, we return ENOMEM. + return ax_err!(NoMemory); + } + let backend = area.backend(); if let Backend::Alloc { populate, align } = *backend { - if !populate { - for addr in PageIterWrapper::new(start, area.end().min(end), align).unwrap() { - match self.pt.query(addr) { - Ok(_) => {} - // If the page is not mapped, try map it. - Err(PagingError::NotMapped) => { + for addr in PageIterWrapper::new(start, area.end().min(end), align).unwrap() { + match self.pt.query(addr) { + #[allow(unused_variables)] + Ok((paddr, flags, page_size)) => { + #[cfg(feature = "cow")] + { + // if the page is already mapped and write intentions, try cow. + if flags.contains(MappingFlags::WRITE) { + continue; + } else if _access_flags.contains(MappingFlags::WRITE) + && !Self::handle_cow_fault( + addr, + paddr, + flags, + page_size, + &mut self.pt, + ) + { + return Err(AxError::NoMemory); + } + } + } + // If the page is not mapped, try map it. + Err(PagingError::NotMapped) => { + if !populate { if !backend.handle_page_fault(addr, area.flags(), &mut self.pt) { return Err(AxError::NoMemory); } + } else { + return Err(AxError::BadAddress); } - Err(_) => return Err(AxError::BadAddress), - }; - } + } + Err(_) => return Err(AxError::BadAddress), + }; } } start = area.end(); - assert!(start.is_aligned(align)); + assert!(start.is_aligned(PageSize::Size4K)); if start >= end { - break; + return Ok(()); } } - if start < end { - // If the area is not fully mapped, we return ENOMEM. - return ax_err!(NoMemory); - } - - Ok(()) + // start < end + // If the area is not fully mapped, we return ENOMEM. + ax_err!(NoMemory) } /// Removes mappings within the specified virtual address range. @@ -387,15 +444,9 @@ impl AddrSpace { /// /// Returns an error if the address range is out of the address space or not /// aligned. - pub fn protect( - &mut self, - start: VirtAddr, - size: usize, - flags: MappingFlags, - align: PageSize, - ) -> AxResult { + pub fn protect(&mut self, start: VirtAddr, size: usize, flags: MappingFlags) -> AxResult { // Populate the area first, which also checks the address range for us. - self.populate_area(start, size, align)?; + self.populate_area(start, size, flags)?; self.areas .protect(start, size, |_| Some(flags), &mut self.pt) @@ -453,6 +504,25 @@ impl AddrSpace { if let Some(area) = self.areas.find(vaddr) { let orig_flags = area.flags(); if orig_flags.contains(access_flags) { + // Two cases enter the branch: + // - shared pages (If there is a shared page in the vma) + // - cow + #[cfg(feature = "cow")] + if access_flags.contains(MappingFlags::WRITE) + && let Ok((paddr, _, page_size)) = self.pt.query(vaddr) + { + // 1. page fault caused by write + // 2. pte exists + // 3. Not shared memory + return Self::handle_cow_fault( + vaddr, + paddr, + orig_flags, + page_size, + &mut self.pt, + ); + } + return area .backend() .handle_page_fault(vaddr, orig_flags, &mut self.pt); @@ -461,58 +531,175 @@ impl AddrSpace { false } - /// Clone a [`AddrSpace`] by re-mapping all [`MemoryArea`]s in a new page table and copying data in user space. - pub fn clone_or_err(&mut self) -> AxResult { + /// Attempts to clone the current address space into a new one. + /// + /// This method creates a new empty address space with the same base and size, + /// then iterates over all memory areas in the original address space to copy or + /// share their mappings into the new one. + /// + /// ### Behavior with `cow` Feature Enabled + /// - For memory areas backed by [`Backend::Alloc`], the `populate` flag is forced + /// to `false` to avoid preemptive physical allocation in the new space. + /// - All writable mappings have their `WRITE` flag removed, enforcing + /// Copy-On-Write (COW) semantics. + /// - Shared pages increase their reference count via `frame_table().inc_ref()`, + /// and both the original and the cloned page tables are updated: + /// - The original page's protection flags are modified to remove write access. + /// - The new address space maps the same physical page with the new flags. + /// + /// ### Behavior without `cow` Feature + /// - Each mapped page in the original address space is copied into the + /// corresponding address in the new address space. + /// - If the target address in the new space is not mapped, a page fault is + /// handled via [`Backend::handle_page_fault`], and memory is allocated before copying. + /// - The actual copying is done using [`core::ptr::copy_nonoverlapping`] at the + /// physical address level. + pub fn try_clone(&mut self) -> AxResult { let mut new_aspace = Self::new_empty(self.base(), self.size())?; for area in self.areas.iter() { - let backend = area.backend(); + let backend = match area.backend() { + #[cfg(feature = "cow")] + Backend::Alloc { populate: _, align } => { + // Forcing `populate = false` is to prevent the subsequent `new_aspace.areas.map` + // from mapping page table entries for the virtual addresses. + Backend::new_alloc(false, *align) + } + other => other.clone(), + }; + // Remap the memory area in the new address space. - let new_area = - MemoryArea::new(area.start(), area.size(), area.flags(), backend.clone()); + let new_area = MemoryArea::new(area.start(), area.size(), area.flags(), backend); new_aspace .areas .map(new_area, &mut new_aspace.pt, false) .map_err(mapping_err_to_ax_err)?; - if matches!(backend, Backend::Linear { .. }) { - continue; - } - // Copy data from old memory area to new memory area. - for vaddr in - PageIter4K::new(area.start(), area.end()).expect("Failed to create page iterator") + let align = match area.backend() { + Backend::Alloc { align, .. } => *align, + // Linear-backed regions are usually allocated by the kernel and are shared + Backend::Linear { .. } => continue, + }; + + #[cfg(feature = "cow")] + let cow_flags = area.flags() - MappingFlags::WRITE; + + for vaddr in PageIterWrapper::new(area.start(), area.end(), align) + .expect("Failed to create page iterator") { - let addr = match self.pt.query(vaddr) { - Ok((paddr, _, _)) => paddr, - // If the page is not mapped, skip it. - Err(PagingError::NotMapped) => continue, - Err(_) => return Err(AxError::BadAddress), - }; - let new_addr = match new_aspace.pt.query(vaddr) { - Ok((paddr, _, _)) => paddr, - // If the page is not mapped, try map it. - Err(PagingError::NotMapped) => { - if !backend.handle_page_fault(vaddr, area.flags(), &mut new_aspace.pt) { - return Err(AxError::NoMemory); + // Copy data from old memory area to new memory area. + match self.pt.query(vaddr) { + Ok((paddr, _, page_size)) => { + #[cfg(not(feature = "cow"))] + { + let new_addr = match new_aspace.pt.query(vaddr) { + Ok((paddr, _, _)) => paddr, + // If the page is not mapped, try map it. + Err(PagingError::NotMapped) => { + if !area.backend().handle_page_fault( + vaddr, + area.flags(), + &mut new_aspace.pt, + ) { + return Err(AxError::NoMemory); + } + match new_aspace.pt.query(vaddr) { + Ok((paddr, _, _)) => paddr, + Err(_) => return Err(AxError::BadAddress), + } + } + Err(_) => return Err(AxError::BadAddress), + }; + unsafe { + core::ptr::copy_nonoverlapping( + phys_to_virt(paddr).as_ptr(), + phys_to_virt(new_addr).as_mut_ptr(), + page_size.into(), + ) + }; } - match new_aspace.pt.query(vaddr) { - Ok((paddr, _, _)) => paddr, - Err(_) => return Err(AxError::BadAddress), + + //If the page is mapped in the old page table: + // - Update its permissions in the old page table using `flags`. + // - Map the same physical page into the new page table at the same + // virtual address, with the same page size and `flags`. + #[cfg(feature = "cow")] + { + frame_table().inc_ref(paddr); + + self.pt + .protect(vaddr, cow_flags) + .map(|(_, tlb)| tlb.flush()) + .expect("protect failed"); + new_aspace + .pt + .map(vaddr, paddr, page_size, cow_flags) + .map(|tlb| tlb.flush()) + .expect("map failed"); + + continue; } } + // If the page is not mapped, skip it. + Err(PagingError::NotMapped) => continue, Err(_) => return Err(AxError::BadAddress), }; - unsafe { - core::ptr::copy_nonoverlapping( - phys_to_virt(addr).as_ptr(), - phys_to_virt(new_addr).as_mut_ptr(), - PAGE_SIZE_4K, - ) - }; } } Ok(new_aspace) } + + /// Handles a Copy-On-Write (COW) page fault. + /// + /// # Arguments + /// - `vaddr`: The virtual address that triggered the fault. + /// - `paddr`: The physical address that triggered the fault. + /// - `flags`: vma flags. + /// - `align`: Alignment requirement for the allocated memory, must be a multiple of 4KiB. + /// - `pt`: A mutable reference to the page table that should be updated. + /// + /// # Returns + /// - `true` if the page fault was handled successfully. + /// - `false` if the fault handling failed (e.g., allocation failed or invalid ref count). + #[cfg(feature = "cow")] + fn handle_cow_fault( + vaddr: VirtAddr, + paddr: PhysAddr, + flags: MappingFlags, + align: PageSize, + pt: &mut PageTable, + ) -> bool { + let paddr = paddr.align_down(align); + + match frame_table().ref_count(paddr) { + 0 => unreachable!(), + // There is only one AddrSpace reference to the page, + // so there is no need to copy it. + 1 => pt.protect(vaddr, flags).map(|(_, tlb)| tlb.flush()).is_ok(), + // Allocates the new page and copies the contents of the original page, + // remapping the virtual address to the physical address of the new page. + 2.. => match alloc_frame(false, align) { + Some(new_frame) => { + unsafe { + core::ptr::copy_nonoverlapping( + phys_to_virt(paddr).as_ptr(), + phys_to_virt(new_frame).as_mut_ptr(), + align.into(), + ) + }; + + dealloc_frame(paddr, align); + + pt.remap(vaddr, new_frame, flags) + .map(|(_, tlb)| { + tlb.flush(); + }) + .is_ok() + } + None => false, + }, + } + } } impl fmt::Debug for AddrSpace { diff --git a/modules/axmm/src/backend/alloc.rs b/modules/axmm/src/backend/alloc.rs index 2537e832bf..7cc40bab19 100644 --- a/modules/axmm/src/backend/alloc.rs +++ b/modules/axmm/src/backend/alloc.rs @@ -1,9 +1,12 @@ -use crate::backend::page_iter_wrapper::PageIterWrapper; +use crate::page_iter_wrapper::PageIterWrapper; use axalloc::global_allocator; use axhal::mem::{phys_to_virt, virt_to_phys}; use axhal::paging::{MappingFlags, PageSize, PageTable}; use memory_addr::{PAGE_SIZE_4K, PhysAddr, VirtAddr}; +#[cfg(feature = "cow")] +use crate::frameinfo::frame_table; + use super::Backend; /// Allocates a physical frame, with an option to zero it out. @@ -25,7 +28,7 @@ use super::Backend; /// - If `zeroed` is `true`, the function uses `unsafe` operations to zero out the memory. /// - The allocated memory must be accessed via its physical address, which requires /// conversion using `virt_to_phys`. -fn alloc_frame(zeroed: bool, align: PageSize) -> Option { +pub fn alloc_frame(zeroed: bool, align: PageSize) -> Option { let page_size: usize = align.into(); let num_pages = page_size / PAGE_SIZE_4K; let vaddr = VirtAddr::from(global_allocator().alloc_pages(num_pages, page_size).ok()?); @@ -33,6 +36,10 @@ fn alloc_frame(zeroed: bool, align: PageSize) -> Option { unsafe { core::ptr::write_bytes(vaddr.as_mut_ptr(), 0, page_size) }; } let paddr = virt_to_phys(vaddr); + + #[cfg(feature = "cow")] + frame_table().inc_ref(paddr); + Some(paddr) } @@ -43,6 +50,9 @@ fn alloc_frame(zeroed: bool, align: PageSize) -> Option { /// The size of the memory to be freed is determined by the `align` parameter, /// which must be a multiple of 4KiB. /// +/// If `cow` feature is enabled, this function decreases the reference count associated with the frame. +/// When the reference count reaches 1, it actually frees the frame memory. +/// /// # Parameters /// - `frame`: The physical address of the memory to be freed. /// - `align`: The alignment requirement for the memory, must be a multiple of 4KiB. @@ -52,10 +62,15 @@ fn alloc_frame(zeroed: bool, align: PageSize) -> Option { /// otherwise undefined behavior may occur. /// - If the deallocation fails, the function will call `panic!`. Details about /// the failure can be obtained from the global memory allocator’s error messages. -fn dealloc_frame(frame: PhysAddr, align: PageSize) { +pub fn dealloc_frame(frame: PhysAddr, align: PageSize) { + #[cfg(feature = "cow")] + if frame_table().dec_ref(frame) > 1 { + return; + } + + let vaddr = phys_to_virt(frame); let page_size: usize = align.into(); let num_pages = page_size / PAGE_SIZE_4K; - let vaddr = phys_to_virt(frame); global_allocator().dealloc_pages(vaddr.as_usize(), num_pages); } diff --git a/modules/axmm/src/backend/mod.rs b/modules/axmm/src/backend/mod.rs index 378b685fa0..59c5a45c70 100644 --- a/modules/axmm/src/backend/mod.rs +++ b/modules/axmm/src/backend/mod.rs @@ -3,12 +3,12 @@ use axhal::paging::{MappingFlags, PageTable}; use memory_addr::VirtAddr; use memory_set::MappingBackend; -pub use page_iter_wrapper::PageIterWrapper; use page_table_multiarch::PageSize; mod alloc; mod linear; -mod page_iter_wrapper; + +pub use alloc::{alloc_frame, dealloc_frame}; /// A unified enum type for different memory mapping backends. /// diff --git a/modules/axmm/src/frameinfo.rs b/modules/axmm/src/frameinfo.rs new file mode 100644 index 0000000000..b6add837de --- /dev/null +++ b/modules/axmm/src/frameinfo.rs @@ -0,0 +1,85 @@ +//! FrameInfo +//! +//! A simple physical FrameInfo manager is provided to track and manage +//! the reference count for every 4KB memory page frame in the system. +//! +//! There is a [`FrameInfo`] struct for each physical page frame +//! that keeps track of its reference count. +//! NOTE: If the page is huge page, its [`FrameInfo`] is placed at the +//! starting physical address. +use core::{ + array, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use alloc::boxed::Box; +use lazy_static::lazy_static; +use memory_addr::PhysAddr; +// 4 kb page +const FRAME_SHIFT: usize = 12; + +pub const MAX_FRAME_NUM: usize = axconfig::plat::PHYS_MEMORY_SIZE >> FRAME_SHIFT; + +lazy_static! { + static ref FRAME_INFO_TABLE: FrameRefTable = FrameRefTable::default(); +} + +pub(crate) fn frame_table() -> &'static FrameRefTable { + &FRAME_INFO_TABLE +} + +pub(crate) struct FrameRefTable { + data: Box<[FrameInfo; MAX_FRAME_NUM]>, +} + +impl Default for FrameRefTable { + fn default() -> Self { + FrameRefTable { + data: Box::new(array::from_fn(|_| FrameInfo::default())), + } + } +} + +impl FrameRefTable { + fn info(&self, paddr: PhysAddr) -> &FrameInfo { + let index = (paddr.as_usize() - axconfig::plat::PHYS_MEMORY_BASE) >> FRAME_SHIFT; + &self.data[index] + } + + /// Increases the reference count of the frame associated with a physical address. + /// + /// # Parameters + /// - `paddr`: It must be an aligned physical address; if it's a huge page, + /// it must be the starting physical address. + pub fn inc_ref(&self, paddr: PhysAddr) { + self.info(paddr).ref_count.fetch_add(1, Ordering::SeqCst); + } + + /// Decreases the reference count of the frame associated with a physical address. + /// + /// - `paddr`: It must be an aligned physical address; if it's a huge page, + /// it must be the starting physical address. + /// + /// # Returns + /// The updated reference count after decrementing. + pub fn dec_ref(&self, paddr: PhysAddr) -> usize { + self.info(paddr).ref_count.fetch_sub(1, Ordering::SeqCst) + } + + /// Returns the `FrameInfo` structure associated with a given physical address. + /// + /// # Parameters + /// - `paddr`: It must be an aligned physical address; if it's a huge page, + /// it must be the starting physical address. + /// + /// # Returns + /// A reference to the `FrameInfo` associated with the given physical address. + pub fn ref_count(&self, paddr: PhysAddr) -> usize { + self.info(paddr).ref_count.load(Ordering::SeqCst) + } +} + +#[derive(Default)] +pub(crate) struct FrameInfo { + ref_count: AtomicUsize, +} diff --git a/modules/axmm/src/lib.rs b/modules/axmm/src/lib.rs index ad1db55875..92cf8d086d 100644 --- a/modules/axmm/src/lib.rs +++ b/modules/axmm/src/lib.rs @@ -8,6 +8,9 @@ extern crate alloc; mod aspace; mod backend; +#[cfg(feature = "cow")] +mod frameinfo; +mod page_iter_wrapper; pub use self::aspace::AddrSpace; pub use self::backend::Backend; diff --git a/modules/axmm/src/backend/page_iter_wrapper.rs b/modules/axmm/src/page_iter_wrapper.rs similarity index 99% rename from modules/axmm/src/backend/page_iter_wrapper.rs rename to modules/axmm/src/page_iter_wrapper.rs index 98b5ac33fb..08f56b530e 100644 --- a/modules/axmm/src/backend/page_iter_wrapper.rs +++ b/modules/axmm/src/page_iter_wrapper.rs @@ -56,7 +56,6 @@ impl PageIterWrapper { PageSize::Size4K => PageIter4K::::new(start, end).map(Self::Size4K), PageSize::Size2M => PageIter2M::::new(start, end).map(Self::Size2M), PageSize::Size1G => PageIter1G::::new(start, end).map(Self::Size1G), - _ => None, } } }