The current RcuBox will call the `drop()` function after a grace period
inside an RCU callback. This suffices for maintaining a RCU-protected
object:
RcuBox::drop():
call_rcu(
|..| { // <- call back after one grace period.
T::drop(); // <- call the destructor of the inner object.
}
)
However, to support a different RCU usage pattern as below we need to
extend RcuBox:
1. clean up the object, and unshare it from future RCU readers.
2. wait for an RCU grace period.
3. no other RCU readers, we can free the memory.
An `RcuFreeBox<T: RcuFreeSafe>` is introduced to provide support for
this:
RcuFreeBox::drop():
T::drop_before_gp(); // clean up and ushare.
kfree_call_rcu(..); // free it after one grace period.
Signed-off-by: Boqun Feng <[email protected]>
---
rust/kernel/sync/rcu.rs | 34 ++++++++++++++++
rust/kernel/sync/rcu/rcu_box.rs | 72 +++++++++++++++++++++++++++++++--
2 files changed, 102 insertions(+), 4 deletions(-)
diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs
index 42f6bbc83f71..e781a5044de9 100644
--- a/rust/kernel/sync/rcu.rs
+++ b/rust/kernel/sync/rcu.rs
@@ -4,6 +4,8 @@
//!
//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h)
+use core::pin::Pin;
+
use crate::{
bindings,
types::{
@@ -18,6 +20,8 @@
pub use self::rcu_box::RcuKVBox;
pub use self::rcu_box::RcuVBox;
+pub use self::rcu_box::RcuFreeBox;
+
/// Evidence that the RCU read side lock is held on the current thread/CPU.
///
/// The type is explicitly not `Send` because this property is per-thread/CPU.
@@ -100,3 +104,33 @@ pub trait ForeignOwnableRcu: ForeignOwnable {
/// [`from_foreign`]: ForeignOwnable::from_foreign
unsafe fn rcu_borrow<'a>(ptr: *mut ffi::c_void) -> Self::RcuBorrowed<'a>;
}
+
+/// Declares a struct is safe to free after a grace period if all readers are
guarded by RCU.
+///
+/// # Safety
+///
+/// Implementation must guarantee `drop_before_gp()` makes sure no future RCU
reader will access
+/// any part of [`Self`], as a result, after `drop_before_gp()` return + one
grace period, no RCU
+/// reader will be on the object, and it's safe to free it.
+///
+/// Notes for implementators: implementing this trait in general requires
`Self` being a
+/// [`UnsafePinned`], i.e. a `&mut Self` is not a noalias reference if `Self`
has non-trivial
+/// `drop()` function.
+pub unsafe trait RcuFreeSafe {
+ /// Clean up `Self` and make it ready to be RCU freed.
+ fn drop_before_gp(self: Pin<&mut Self>);
+}
+
+macro_rules! impl_not_drop {
+ ($($t:ty, )*) => {
+ // SAFETY: Dropping `T` has no side effect means `T` is always ready
to be freed. And an
+ // empty `drop_before_gp()` suffices.
+ $(unsafe impl RcuFreeSafe for $t {
+ fn drop_before_gp(self: Pin<&mut Self>) {
+ $crate::const_assert!(!core::mem::needs_drop::<$t>());
+ }
+ })*
+ }
+}
+
+impl_not_drop! {i8,u8,i16,u16,i32,u32,isize,usize,i64,u64,}
diff --git a/rust/kernel/sync/rcu/rcu_box.rs b/rust/kernel/sync/rcu/rcu_box.rs
index cb1fd422480a..ab4e9b2a4444 100644
--- a/rust/kernel/sync/rcu/rcu_box.rs
+++ b/rust/kernel/sync/rcu/rcu_box.rs
@@ -6,6 +6,7 @@
use core::{
marker::PhantomData,
+ mem::ManuallyDrop,
ops::Deref,
ptr::NonNull, //
};
@@ -29,17 +30,18 @@
use super::{
ForeignOwnableRcu,
- Guard, //
+ Guard,
+ RcuFreeSafe, //
};
-/// A box that is freed with rcu.
+/// A box that is drop with RCU.
///
-/// The value must be `Send`, as rcu may drop it on another thread.
+/// The value must be `Send`, as RCU may drop it on another thread.
///
/// # Invariants
///
/// * The pointer is valid and references a pinned `RcuBoxInner<T>` allocated
with `A`.
-/// * This `RcuBox` holds exclusive permissions to rcu free the allocation.
+/// * This `RcuBox` holds exclusive permissions to RCU-free the allocation.
pub struct RcuBox<T: Send, A: Allocator>(NonNull<RcuBoxInner<T>>,
PhantomData<A>);
/// Type alias for [`RcuBox`] with a [`Kmalloc`] allocator.
@@ -223,6 +225,56 @@ fn drop(&mut self) {
drop(unsafe { Box::<_, A>::from_raw(box_inner) });
}
+/// A box that is freed with RCU.
+///
+/// Currently we require `T` being `Send` because of an implementation
limitation. In theory we can
+/// support `T` being `!Send`, since the RCU callback is only used to free the
memory, not dropping
+/// `T`.
+pub struct RcuFreeBox<T: Send + RcuFreeSafe, A:
Allocator>(RcuBox<ManuallyDrop<T>, A>);
+
+impl<T: Send + RcuFreeSafe, A: Allocator> RcuFreeBox<T, A> {
+ /// Create a new `RcuFreeBox`.
+ pub fn new(x: T, flags: alloc::Flags) -> Result<Self, AllocError> {
+ Ok(Self(RcuBox::new(ManuallyDrop::new(x), flags)?))
+ }
+}
+
+impl<T: Send + RcuFreeSafe, A: Allocator> Deref for RcuFreeBox<T, A> {
+ type Target = T;
+
+ fn deref(&self) -> &T {
+ self.0.deref()
+ }
+}
+
+impl<T: Send + RcuFreeSafe, A: Allocator> Drop for RcuFreeBox<T, A> {
+ fn drop(&mut self) {
+ // CAST: `ManuallyDrop<RcuBoxInner<T>>` is transparent to
`RcuBoxInner<T>`, and `RcuBox`
+ // owns the object per type invariants.
+ let inner: *mut RcuBoxInner<T> = self.0 .0.as_ptr().cast();
+
+ // SAFETY: Per the invariants of `RcuBox`, `inner` owns the pointed
object. And we are not
+ // going to move it.
+ let pin = unsafe { Pin::new_unchecked(&mut (*inner).value) };
+
+ pin.drop_before_gp();
+
+ // `needs_drop::<ManuallyDrop>()` returns `false`, hence
`kvfree_call_rcu()` will be called
+ // and free the underlying data after a grace period.
+ }
+}
+
+// Note that `T: Sync` is required since when moving an `RcuFreeBox<T, A>`,
the previous owner may
+// still access `&T` for one grace period.
+//
+// SAFETY: Ownership of the `RcuFreeBox<T, A>` allows for `&T` and dropping
the `T`, so `T: Send +
+// Sync` implies `RcuFreeBox<T, A>: Send`.
+unsafe impl<T: Send + Sync + RcuFreeSafe, A: Allocator> Send for RcuFreeBox<T,
A> {}
+
+// SAFETY: `&RcuFreeBox<T, A>` allows for no operations other than those
permitted by `&T`, so `T:
+// Sync` implies `RcuFreeBox<T, A>: Sync`.
+unsafe impl<T: Send + Sync + RcuFreeSafe, A: Allocator> Sync for RcuFreeBox<T,
A> {}
+
#[kunit_tests(rust_rcu_box)]
mod tests {
use super::*;
@@ -236,6 +288,12 @@ fn rcu_box_basic() -> Result {
drop(rb);
+ let rb = RcuFreeBox::<_, alloc::allocator::Kmalloc>::new(42i32,
alloc::flags::GFP_KERNEL)?;
+
+ assert_eq!(*rb, 42);
+
+ drop(rb);
+
let rb = RcuBox::<_, alloc::allocator::Vmalloc>::new(42i32,
alloc::flags::GFP_KERNEL)?;
assert_eq!(*rb, 42);
@@ -243,6 +301,12 @@ fn rcu_box_basic() -> Result {
drop(rb);
+ let rb = RcuFreeBox::<_, alloc::allocator::Vmalloc>::new(42i32,
alloc::flags::GFP_KERNEL)?;
+
+ assert_eq!(*rb, 42);
+
+ drop(rb);
+
Ok(())
}
}
--
2.50.1 (Apple Git-155)
