diff --git a/drivers/android/process.rs b/drivers/android/process.rs index 60507b73a7c528..b47da6cffaa67b 100644 --- a/drivers/android/process.rs +++ b/drivers/android/process.rs @@ -14,7 +14,7 @@ use kernel::{ linked_list::List, pages::Pages, prelude::*, - sync::{Guard, Mutex, Ref, RefCount, RefCounted}, + sync::{Guard, Mutex, Ref}, user_ptr::{UserSlicePtr, UserSlicePtrReader}, Error, }; @@ -275,7 +275,6 @@ impl ProcessNodeRefs { pub(crate) struct Process { ctx: Arc, - ref_count: RefCount, // TODO: For now this a mutex because we have allocations in BTreeMap and RangeAllocator while // holding the lock. We may want to split up the process state at some point to use a spin lock @@ -293,22 +292,23 @@ unsafe impl Sync for Process {} impl Process { fn new(ctx: Arc) -> Result> { - let mut proc_ref = Ref::try_new(Self { - ref_count: RefCount::new(), - ctx, - // SAFETY: `inner` is initialised in the call to `mutex_init` below. - inner: unsafe { Mutex::new(ProcessInner::new()) }, - // SAFETY: `node_refs` is initialised in the call to `mutex_init` below. - node_refs: unsafe { Mutex::new(ProcessNodeRefs::new()) }, - })?; - let process = Ref::get_mut(&mut proc_ref).ok_or(Error::EINVAL)?; - // SAFETY: `inner` is pinned behind the `Arc` reference. - let pinned = unsafe { Pin::new_unchecked(&process.inner) }; - kernel::mutex_init!(pinned, "Process::inner"); - // SAFETY: `node_refs` is pinned behind the `Arc` reference. - let pinned = unsafe { Pin::new_unchecked(&process.node_refs) }; - kernel::mutex_init!(pinned, "Process::node_refs"); - Ok(proc_ref) + Ref::try_new_and_init( + Self { + ctx, + // SAFETY: `inner` is initialised in the call to `mutex_init` below. + inner: unsafe { Mutex::new(ProcessInner::new()) }, + // SAFETY: `node_refs` is initialised in the call to `mutex_init` below. + node_refs: unsafe { Mutex::new(ProcessNodeRefs::new()) }, + }, + |process| { + // SAFETY: `inner` is pinned behind the `Ref` reference. + let pinned = unsafe { Pin::new_unchecked(&process.inner) }; + kernel::mutex_init!(pinned, "Process::inner"); + // SAFETY: `node_refs` is pinned behind the `Ref` reference. + let pinned = unsafe { Pin::new_unchecked(&process.node_refs) }; + kernel::mutex_init!(pinned, "Process::node_refs"); + }, + ) } /// Attemps to fetch a work item from the process queue. @@ -337,7 +337,7 @@ impl Process { Either::Right(Registration::new(self, thread, &mut inner)) } - fn get_thread(&self, id: i32) -> Result> { + fn get_thread(self: &Ref, id: i32) -> Result> { // TODO: Consider using read/write locks here instead. { let inner = self.inner.lock(); @@ -347,7 +347,7 @@ impl Process { } // Allocate a new `Thread` without holding any locks. - let ta = Thread::new(id, Ref::new_from(self))?; + let ta = Thread::new(id, self.clone())?; let mut inner = self.inner.lock(); @@ -366,7 +366,7 @@ impl Process { self.inner.lock().push_work(work) } - fn set_as_manager(&self, info: Option, thread: &Thread) -> Result { + fn set_as_manager(self: &Ref, info: Option, thread: &Thread) -> Result { let (ptr, cookie, flags) = if let Some(obj) = info { ( // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it @@ -390,7 +390,7 @@ impl Process { } pub(crate) fn get_node( - &self, + self: &Ref, ptr: usize, cookie: usize, flags: u32, @@ -406,7 +406,7 @@ impl Process { } // Allocate the node before reacquiring the lock. - let node = Arc::try_new(Node::new(ptr, cookie, flags, Ref::new_from(self)))?; + let node = Arc::try_new(Node::new(ptr, cookie, flags, self.clone()))?; let mut inner = self.inner.lock(); if let Some(node) = inner.get_existing_node_ref(ptr, cookie, strong, thread)? { @@ -693,7 +693,11 @@ impl Process { ret } - pub(crate) fn request_death(&self, reader: &mut UserSlicePtrReader, thread: &Thread) -> Result { + pub(crate) fn request_death( + self: &Ref, + reader: &mut UserSlicePtrReader, + thread: &Thread, + ) -> Result { let handle: u32 = reader.read()?; let cookie: usize = reader.read()?; @@ -716,9 +720,8 @@ impl Process { } // SAFETY: `init` is called below. - let death = death.commit(unsafe { - NodeDeath::new(info.node_ref.node.clone(), Ref::new_from(self), cookie) - }); + let death = death + .commit(unsafe { NodeDeath::new(info.node_ref.node.clone(), self.clone(), cookie) }); // SAFETY: `death` is pinned behind the `Arc` reference. unsafe { Pin::new_unchecked(death.as_ref()) }.init(); info.death = Some(death.clone()); @@ -766,9 +769,14 @@ impl Process { } impl IoctlHandler for Process { - type Target = Self; - - fn write(this: &Self, _file: &File, cmd: u32, reader: &mut UserSlicePtrReader) -> Result { + type Target = Ref; + + fn write( + this: &Ref, + _file: &File, + cmd: u32, + reader: &mut UserSlicePtrReader, + ) -> Result { let thread = this.get_thread(unsafe { rust_helper_current_pid() })?; match cmd { bindings::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?), @@ -782,7 +790,7 @@ impl IoctlHandler for Process { Ok(0) } - fn read_write(this: &Self, file: &File, cmd: u32, data: UserSlicePtr) -> Result { + fn read_write(this: &Ref, file: &File, cmd: u32, data: UserSlicePtr) -> Result { let thread = this.get_thread(unsafe { rust_helper_current_pid() })?; match cmd { bindings::BINDER_WRITE_READ => thread.write_read(data, file.is_blocking())?, @@ -795,12 +803,6 @@ impl IoctlHandler for Process { } } -unsafe impl RefCounted for Process { - fn get_count(&self) -> &RefCount { - &self.ref_count - } -} - impl FileOpener> for Process { fn open(ctx: &Arc) -> Result { let process = Self::new(ctx.clone())?; @@ -893,15 +895,15 @@ impl FileOperations for Process { } } - fn ioctl(this: &Process, file: &File, cmd: &mut IoctlCommand) -> Result { + fn ioctl(this: &Ref, file: &File, cmd: &mut IoctlCommand) -> Result { cmd.dispatch::(this, file) } - fn compat_ioctl(this: &Process, file: &File, cmd: &mut IoctlCommand) -> Result { + fn compat_ioctl(this: &Ref, file: &File, cmd: &mut IoctlCommand) -> Result { cmd.dispatch::(this, file) } - fn mmap(this: &Process, _file: &File, vma: &mut bindings::vm_area_struct) -> Result { + fn mmap(this: &Ref, _file: &File, vma: &mut bindings::vm_area_struct) -> Result { // TODO: Only group leader is allowed to create mappings. if vma.vm_start == 0 { @@ -919,7 +921,7 @@ impl FileOperations for Process { this.create_mapping(vma) } - fn poll(this: &Process, file: &File, table: &PollTable) -> Result { + fn poll(this: &Ref, file: &File, table: &PollTable) -> Result { let thread = this.get_thread(unsafe { rust_helper_current_pid() })?; let (from_proc, mut mask) = thread.poll(file, table); if mask == 0 && from_proc && !this.inner.lock().work.is_empty() { diff --git a/rust/helpers.c b/rust/helpers.c index 5450392e6036f2..749ad1dafa3c4e 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -146,6 +146,24 @@ rust_helper_platform_set_drvdata(struct platform_device *pdev, } EXPORT_SYMBOL_GPL(rust_helper_platform_set_drvdata); +refcount_t rust_helper_refcount_new(void) +{ + return (refcount_t)REFCOUNT_INIT(1); +} +EXPORT_SYMBOL_GPL(rust_helper_refcount_new); + +void rust_helper_refcount_inc(refcount_t *r) +{ + refcount_inc(r); +} +EXPORT_SYMBOL_GPL(rust_helper_refcount_inc); + +bool rust_helper_refcount_dec_and_test(refcount_t *r) +{ + return refcount_dec_and_test(r); +} +EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test); + /* We use bindgen's --size_t-is-usize option to bind the C size_t type * as the Rust usize type, so we can use it in contexts where Rust * expects a usize like slice (array) indices. usize is defined to be diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 7ceb8985f83656..45467e38e2d3d5 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -21,6 +21,7 @@ const_panic, const_raw_ptr_deref, const_unreachable_unchecked, + receiver_trait, try_reserve )] #![deny(clippy::complexity)] diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 301d25c144db00..ddecb9e371a45c 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -7,21 +7,27 @@ //! underlying object when it reaches zero. It is also safe to use concurrently from multiple //! threads. //! -//! It is different from the standard library's [`Arc`] in two ways: it does not support weak -//! references, which allows it to be smaller -- a single pointer-sized integer; it allows users to -//! safely increment the reference count from a single reference to the underlying object. +//! It is different from the standard library's [`Arc`] in a few ways: +//! 1. It is backed by the kernel's `refcount_t` type. +//! 2. It does not support weak references, which allows it to be half the size. +//! 3. It saturates the reference count instead of aborting when it goes over a threshold. +//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. //! //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html -use crate::Result; +use crate::{bindings, Result}; use alloc::boxed::Box; use core::{ - mem::ManuallyDrop, - ops::Deref, + cell::UnsafeCell, convert::AsRef, marker::PhantomData, mem::ManuallyDrop, ops::Deref, pin::Pin, ptr::NonNull, - sync::atomic::{fence, AtomicUsize, Ordering}, }; +extern "C" { + fn rust_helper_refcount_new() -> bindings::refcount_t; + fn rust_helper_refcount_inc(r: *mut bindings::refcount_t); + fn rust_helper_refcount_dec_and_test(r: *mut bindings::refcount_t) -> bool; +} + /// A reference-counted pointer to an instance of `T`. /// /// The reference count is incremented when new instances of [`Ref`] are created, and decremented @@ -29,156 +35,193 @@ use core::{ /// /// # Invariants /// -/// The value stored in [`RefCounted::get_count`] corresponds to the number of instances of [`Ref`] -/// that point to that instance of `T`. -pub struct Ref { - ptr: NonNull, +/// The reference count on an instance of [`Ref`] is always non-zero. +/// The object pointed to by [`Ref`] is always pinned. +pub struct Ref { + ptr: NonNull>, + _p: PhantomData>, +} + +struct RefInner { + refcount: UnsafeCell, + data: T, } +// This is to allow [`Ref`] (and variants) to be used as the type of `self`. +impl core::ops::Receiver for Ref {} + // SAFETY: It is safe to send `Ref` to another thread when the underlying `T` is `Sync` because // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs // `T` to be `Send` because any thread that has a `Ref` may ultimately access `T` directly, for // example, when the reference count reaches zero and `T` is dropped. -unsafe impl Send for Ref {} +unsafe impl Send for Ref {} // SAFETY: It is safe to send `&Ref` to another thread when the underlying `T` is `Sync` for // the same reason as above. `T` needs to be `Send` as well because a thread can clone a `&Ref` // into a `Ref`, which may lead to `T` being accessed by the same reasoning as above. -unsafe impl Sync for Ref {} +unsafe impl Sync for Ref {} -impl Ref { +impl Ref { /// Constructs a new reference counted instance of `T`. pub fn try_new(contents: T) -> Result { - let boxed = Box::try_new(contents)?; - boxed.get_count().count.store(1, Ordering::Relaxed); - let ptr = NonNull::from(Box::leak(boxed)); - Ok(Ref { ptr }) + Self::try_new_and_init(contents, |_| {}) } -} -impl Ref { - /// Creates a new reference-counted pointer to the given instance of `T`. + /// Constructs a new reference counted instance of `T` and calls the initialisation function. /// - /// It works by incrementing the current reference count as part of constructing the new - /// pointer. - pub fn new_from(obj: &T) -> Self { - let ref_count = obj.get_count(); - let cur = ref_count.count.fetch_add(1, Ordering::Relaxed); - if cur == usize::MAX { - panic!("Reference count overflowed"); - } - Self { - ptr: NonNull::from(obj), - } - } - - /// Returns a mutable reference to `T` iff the reference count is one. Otherwise returns - /// [`None`]. - pub fn get_mut(&mut self) -> Option<&mut T> { - // Synchronises with the decrement in `drop`. - if self.get_count().count.load(Ordering::Acquire) != 1 { - return None; - } - // SAFETY: Since there is only one reference, we know it isn't possible for another thread - // to concurrently call this. - Some(unsafe { self.ptr.as_mut() }) + /// This is useful because it provides a mutable reference to `T` at its final location. + pub fn try_new_and_init)>(contents: T, init: U) -> Result { + // INVARIANT: The refcount is initialised to a non-zero value. + let mut inner = Box::try_new(RefInner { + // SAFETY: Just an FFI call that returns a `refcount_t` initialised to 1. + refcount: UnsafeCell::new(unsafe { rust_helper_refcount_new() }), + data: contents, + })?; + + // SAFETY: By the invariant, `RefInner` is pinned and `T` is also pinned. + let pinned = unsafe { Pin::new_unchecked(&mut inner.data) }; + + // INVARIANT: The only places where `&mut T` is available are here, which is explicitly + // pinned, and in `drop`. Both are compatible with the pin requirements. + init(pinned); + + Ok(Ref { + ptr: NonNull::from(Box::leak(inner)), + _p: PhantomData, + }) } - /// Determines if two reference-counted pointers point to the same underlying instance of `T`. - pub fn ptr_eq(a: &Self, b: &Self) -> bool { - core::ptr::eq(a.ptr.as_ptr(), b.ptr.as_ptr()) + /// Deconstructs a [`Ref`] object into a `usize`. + /// + /// It can be reconstructed once via [`Ref::from_usize`]. + pub fn into_usize(obj: Self) -> usize { + ManuallyDrop::new(obj).ptr.as_ptr() as _ } - /// Deconstructs a [`Ref`] object into a raw pointer. + /// Borrows a [`Ref`] instance previously deconstructed via [`Ref::into_usize`]. + /// + /// # Safety /// - /// It can be reconstructed once via [`Ref::from_raw`]. - pub fn into_raw(obj: Self) -> *const T { - let no_drop = ManuallyDrop::new(obj); - no_drop.ptr.as_ptr() + /// `encoded` must have been returned by a previous call to [`Ref::into_usize`]. Additionally, + /// [`Ref::from_usize`] can only be called after *all* instances of [`RefBorrow`] have been + /// dropped. + pub unsafe fn borrow_usize(encoded: usize) -> RefBorrow { + // SAFETY: By the safety requirement of this function, we know that `encoded` came from + // a previous call to `Ref::into_usize`. + let obj = ManuallyDrop::new(unsafe { Ref::from_usize(encoded) }); + + // SAFEY: The safety requirements ensure that the object remains alive for the lifetime of + // the returned value. There is no way to create mutable references to the object. + unsafe { RefBorrow::new(obj) } } - /// Recreates a [`Ref`] instance previously deconstructed via [`Ref::into_raw`]. + /// Recreates a [`Ref`] instance previously deconstructed via [`Ref::into_usize`]. /// /// # Safety /// - /// `ptr` must have been returned by a previous call to [`Ref::into_raw`]. Additionally, it - /// can only be called once for each previous call to [``Ref::into_raw`]. - pub unsafe fn from_raw(ptr: *const T) -> Self { + /// `encoded` must have been returned by a previous call to [`Ref::into_usize`]. Additionally, + /// it can only be called once for each previous call to [``Ref::into_usize`]. + pub unsafe fn from_usize(encoded: usize) -> Self { Ref { - ptr: NonNull::new(ptr as _).unwrap(), + ptr: NonNull::new(encoded as _).unwrap(), + _p: PhantomData, } } } -impl Deref for Ref { +impl Ref { + /// Determines if two reference-counted pointers point to the same underlying instance of `T`. + pub fn ptr_eq(a: &Self, b: &Self) -> bool { + core::ptr::eq(a.ptr.as_ptr(), b.ptr.as_ptr()) + } + + /// Returns a pinned version of a given `Ref` instance. + pub fn pinned(obj: Self) -> Pin { + // SAFETY: The type invariants guarantee that the value is pinned. + unsafe { Pin::new_unchecked(obj) } + } +} + +impl Deref for Ref { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to dereference it. - unsafe { self.ptr.as_ref() } + unsafe { &self.ptr.as_ref().data } } } -impl Clone for Ref { +impl Clone for Ref { fn clone(&self) -> Self { - Self::new_from(self) + // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to increment the refcount. + unsafe { rust_helper_refcount_inc(self.ptr.as_ref().refcount.get()) }; + Self { + ptr: self.ptr, + _p: PhantomData, + } + } +} + +impl AsRef for Ref { + fn as_ref(&self) -> &T { + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to dereference it. + unsafe { &self.ptr.as_ref().data } } } -impl Drop for Ref { +impl Drop for Ref { fn drop(&mut self) { - { - // SAFETY: By the type invariant, there is necessarily a reference to the object. - let obj = unsafe { self.ptr.as_ref() }; - - // Synchronises with the acquire below or with the acquire in `get_mut`. - if obj.get_count().count.fetch_sub(1, Ordering::Release) != 1 { - return; - } + // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot + // touch `refcount` after it's decremented to a non-zero value because another thread/CPU + // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to + // freed/invalid memory as long as it is never dereferenced. + let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); + + // INVARIANT: If the refcount reaches zero, there are no other instances of `Ref`, and + // this instance is being dropped, so the broken invariant is not observable. + // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. + let is_zero = unsafe { rust_helper_refcount_dec_and_test(refcount) }; + if is_zero { + // The count reached zero, we must free the memory. + // + // SAFETY: The pointer was initialised from the result of `Box::leak`. + unsafe { Box::from_raw(self.ptr.as_ptr()) }; } - - // Synchronises with the release when decrementing above. This ensures that modifications - // from all previous threads/CPUs are visible to the underlying object's `drop`. - fence(Ordering::Acquire); - - // The count reached zero, we must free the memory. - // - // SAFETY: The pointer was initialised from the result of `Box::into_raw`. - unsafe { Box::from_raw(self.ptr.as_ptr()) }; } } -/// Trait for reference counted objects. +/// A borrowed [`Ref`] with manually-managed lifetime. /// -/// # Safety -/// -/// Implementers of [`RefCounted`] must ensure that all of their constructors call -/// [`Ref::try_new`]. -pub unsafe trait RefCounted { - /// Returns a pointer to the object field holds the reference count. - fn get_count(&self) -> &RefCount; -} - -/// Holds the reference count of an object. +/// # Invariants /// -/// It is meant to be embedded in objects to be reference-counted, with [`RefCounted::get_count`] -/// returning a reference to it. -pub struct RefCount { - count: AtomicUsize, +/// There are no mutable references to the underlying [`Ref`], and it remains valid for the lifetime +/// of the [`RefBorrow`] instance. +pub struct RefBorrow { + inner_ref: ManuallyDrop>, } -impl RefCount { - /// Constructs a new instance of [`RefCount`]. - pub fn new() -> Self { - Self { - count: AtomicUsize::new(1), - } +impl RefBorrow { + /// Creates a new [`RefBorrow`] instance. + /// + /// # Safety + /// + /// Callers must ensure the following for the lifetime of the returned [`RefBorrow`] instance: + /// 1. That `obj` remains valid; + /// 2. That no mutable references to `obj` are created. + unsafe fn new(obj: ManuallyDrop>) -> Self { + // INVARIANT: The safety requirements guarantee the invariants. + Self { inner_ref: obj } } } -impl Default for RefCount { - fn default() -> Self { - Self::new() +impl Deref for RefBorrow { + type Target = Ref; + + fn deref(&self) -> &Self::Target { + self.inner_ref.deref() } } diff --git a/rust/kernel/sync/mod.rs b/rust/kernel/sync/mod.rs index 946d0c4ec7d221..b3ceb6555e8316 100644 --- a/rust/kernel/sync/mod.rs +++ b/rust/kernel/sync/mod.rs @@ -29,7 +29,7 @@ mod locked_by; mod mutex; mod spinlock; -pub use arc::{Ref, RefCount, RefCounted}; +pub use arc::{Ref, RefBorrow}; pub use condvar::CondVar; pub use guard::{Guard, Lock}; pub use locked_by::LockedBy; diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 821ccd4b571f4a..2146ee819cc996 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,7 +6,7 @@ use crate::{ bindings, c_types, - sync::{Ref, RefCounted}, + sync::{Ref, RefBorrow}, }; use alloc::{boxed::Box, sync::Arc}; use core::{ops::Deref, pin::Pin, ptr::NonNull}; @@ -81,24 +81,22 @@ impl PointerWrapper for Box { } } -impl PointerWrapper for Ref { - type Borrowed = UnsafeReference; +impl PointerWrapper for Ref { + type Borrowed = RefBorrow; fn into_pointer(self) -> *const c_types::c_void { - Ref::into_raw(self) as _ + Ref::into_usize(self) as _ } unsafe fn borrow(ptr: *const c_types::c_void) -> Self::Borrowed { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements also ensure that the object remains alive for the lifetime of - // the returned value. - unsafe { UnsafeReference::new(&*ptr.cast()) } + // SAFETY: The safety requirements for this function ensure that the underlying object + // remains valid for the lifetime of the returned value. + unsafe { Ref::borrow_usize(ptr as _) } } unsafe fn from_pointer(ptr: *const c_types::c_void) -> Self { // SAFETY: The passed pointer comes from a previous call to [`Self::into_pointer()`]. - unsafe { Ref::from_raw(ptr as _) } + unsafe { Ref::from_usize(ptr as _) } } }